diff --git a/.github/workflows/commit-access-review.py b/.github/workflows/commit-access-review.py
new file mode 100644
index 000000000000000..97589c138c0b707
--- /dev/null
+++ b/.github/workflows/commit-access-review.py
@@ -0,0 +1,418 @@
+#!/usr/bin/env python3
+# ===-- commit-access-review.py  --------------------------------------------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===------------------------------------------------------------------------===#
+#
+# ===------------------------------------------------------------------------===#
+
+import datetime
+import github
+import re
+import requests
+import time
+import sys
+import re
+
+
+class User:
+    THRESHOLD = 5
+
+    def __init__(self, name, triage_list):
+        self.name = name
+        self.authored = 0
+        self.merged = 0
+        self.reviewed = 0
+        self.triage_list = triage_list
+
+    def add_authored(self, val=1):
+        self.authored += val
+        if self.meets_threshold():
+            print(self.name, "meets the threshold with authored commits")
+            del self.triage_list[self.name]
+
+    def set_authored(self, val):
+        self.authored = 0
+        self.add_authored(val)
+
+    def add_merged(self, val=1):
+        self.merged += val
+        if self.meets_threshold():
+            print(self.name, "meets the threshold with merged commits")
+            del self.triage_list[self.name]
+
+    def add_reviewed(self, val=1):
+        self.reviewed += val
+        if self.meets_threshold():
+            print(self.name, "meets the threshold with reviewed commits")
+            del self.triage_list[self.name]
+
+    def get_total(self):
+        return self.authored + self.merged + self.reviewed
+
+    def meets_threshold(self):
+        return self.get_total() >= self.THRESHOLD
+
+    def __repr__(self):
+        return "{} : a: {} m: {} r: {}".format(
+            self.name, self.authored, self.merged, self.reviewed
+        )
+
+
+def run_graphql_query(
+    query: str, variables: dict, token: str, retry: bool = True
+) -> dict:
+    """
+    This function submits a graphql query and returns the results as a
+    dictionary.
+    """
+    s = requests.Session()
+    retries = requests.adapters.Retry(total=8, backoff_factor=2, status_forcelist=[504])
+    s.mount("https://", requests.adapters.HTTPAdapter(max_retries=retries))
+
+    headers = {
+        "Authorization": "bearer {}".format(token),
+        # See
+        # https://github.blog/2021-11-16-graphql-global-id-migration-update/
+        "X-Github-Next-Global-ID": "1",
+    }
+    request = s.post(
+        url="https://api.github.com/graphql",
+        json={"query": query, "variables": variables},
+        headers=headers,
+    )
+
+    rate_limit = request.headers.get("X-RateLimit-Remaining")
+    print(rate_limit)
+    if rate_limit and int(rate_limit) < 10:
+        reset_time = int(request.headers["X-RateLimit-Reset"])
+        while reset_time - int(time.time()) > 0:
+            time.sleep(60)
+            print(
+                "Waiting until rate limit reset",
+                reset_time - int(time.time()),
+                "seconds remaining",
+            )
+
+    if request.status_code == 200:
+        if "data" not in request.json():
+            print(request.json())
+            sys.exit(1)
+        return request.json()["data"]
+    elif retry:
+        return run_graphql_query(query, variables, token, False)
+    else:
+        raise Exception(
+            "Failed to run graphql query\nquery: {}\nerror: {}".format(
+                query, request.json()
+            )
+        )
+
+
+def check_manual_requests(start_date: datetime.datetime, token: str) -> list[str]:
+    """
+    Return a list of users who have been asked since ``start_date`` if they
+    want to keep their commit access.
+    """
+    query = """
+        query ($query: String!) {
+          search(query: $query, type: ISSUE, first: 100) {
+            nodes {
+              ... on Issue {
+                body
+                comments (first: 100) {
+                  nodes {
+                    author {
+                      login
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+        """
+    formatted_start_date = start_date.strftime("%Y-%m-%dT%H:%M:%S")
+    variables = {
+        "query": f"type:issue created:>{formatted_start_date} org:llvm repo:llvm-project label:infrastructure:commit-access"
+    }
+
+    data = run_graphql_query(query, variables, token)
+    users = []
+    for issue in data["search"]["nodes"]:
+        users.extend([user[1:] for user in re.findall("@[^ ,\n]+", issue["body"])])
+
+    return users
+
+
+def get_num_commits(user: str, start_date: datetime.datetime, token: str) -> int:
+    """
+    Get number of commits that ``user`` has been made since ``start_date`.
+    """
+    variables = {
+        "owner": "llvm",
+        "user": user,
+        "start_date": start_date.strftime("%Y-%m-%dT%H:%M:%S"),
+    }
+
+    user_query = """
+        query ($user: String!) {
+          user(login: $user) {
+            id
+          }
+        }
+    """
+
+    data = run_graphql_query(user_query, variables, token)
+    variables["user_id"] = data["user"]["id"]
+
+    query = """
+        query ($owner: String!, $user_id: ID!, $start_date: GitTimestamp!){
+          organization(login: $owner) {
+            teams(query: "llvm-committers" first:1) {
+              nodes {
+                repositories {
+                  nodes {
+                    ref(qualifiedName: "main") {
+                      target {
+                        ... on Commit {
+                          history(since: $start_date, author: {id: $user_id }) {
+                            totalCount
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+     """
+    count = 0
+    data = run_graphql_query(query, variables, token)
+    for repo in data["organization"]["teams"]["nodes"][0]["repositories"]["nodes"]:
+        count += int(repo["ref"]["target"]["history"]["totalCount"])
+        if count >= User.THRESHOLD:
+            break
+    return count
+
+
+def is_new_committer_query_repo(
+    user: str, start_date: datetime.datetime, token: str
+) -> bool:
+    """
+    Determine if ``user`` is a new committer.  A new committer can keep their
+    commit access even if they don't meet the criteria.
+    """
+    variables = {
+        "user": user,
+    }
+
+    user_query = """
+        query ($user: String!) {
+          user(login: $user) {
+            id
+          }
+        }
+    """
+
+    data = run_graphql_query(user_query, variables, token)
+    variables["owner"] = "llvm"
+    variables["user_id"] = data["user"]["id"]
+    variables["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%S")
+
+    query = """
+        query ($owner: String!, $user_id: ID!){
+          organization(login: $owner) {
+            repository(name: "llvm-project") {
+              ref(qualifiedName: "main") {
+                target {
+                  ... on Commit {
+                    history(author: {id: $user_id }, first: 5) {
+                      nodes {
+                        committedDate
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+     """
+
+    data = run_graphql_query(query, variables, token)
+    repo = data["organization"]["repository"]
+    commits = repo["ref"]["target"]["history"]["nodes"]
+    if len(commits) == 0:
+        return True
+    committed_date = commits[-1]["committedDate"]
+    if datetime.datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%SZ") < start_date:
+        return False
+    return True
+
+
+def is_new_committer(user: str, start_date: datetime.datetime, token: str) -> bool:
+    """
+    Wrapper around is_new_commiter_query_repo to handle exceptions.
+    """
+    try:
+        return is_new_committer_query_repo(user, start_date, token)
+    except:
+        pass
+    return True
+
+
+def get_review_count(user: str, start_date: datetime.datetime, token: str) -> int:
+    """
+    Return the number of reviews that ``user`` has done since ``start_date``.
+    """
+    query = """
+        query ($query: String!) {
+          search(query: $query, type: ISSUE, first: 5) {
+            issueCount
+          }
+        }
+        """
+    formatted_start_date = start_date.strftime("%Y-%m-%dT%H:%M:%S")
+    variables = {
+        "owner": "llvm",
+        "repo": "llvm-project",
+        "user": user,
+        "query": f"type:pr commenter:{user} -author:{user} merged:>{formatted_start_date} org:llvm",
+    }
+
+    data = run_graphql_query(query, variables, token)
+    return int(data["search"]["issueCount"])
+
+
+def count_prs(triage_list: dict, start_date: datetime.datetime, token: str):
+    """
+    Fetch all the merged PRs for the project since ``start_date`` and update
+    ``triage_list`` with the number of PRs merged for each user.
+    """
+
+    query = """
+        query ($query: String!, $after: String) {
+          search(query: $query, type: ISSUE, first: 100, after: $after) {
+            issueCount,
+            nodes {
+              ... on PullRequest {
+                 author {
+                   login
+                 }
+                 mergedBy {
+                   login
+                 }
+              }
+            }
+            pageInfo {
+              hasNextPage
+              endCursor
+            }
+          }
+        }
+    """
+    date_begin = start_date
+    date_end = None
+    while date_begin < datetime.datetime.now():
+        date_end = date_begin + datetime.timedelta(days=7)
+        formatted_date_begin = date_begin.strftime("%Y-%m-%dT%H:%M:%S")
+        formatted_date_end = date_end.strftime("%Y-%m-%dT%H:%M:%S")
+        variables = {
+            "query": f"type:pr is:merged merged:{formatted_date_begin}..{formatted_date_end} org:llvm",
+        }
+        has_next_page = True
+        while has_next_page:
+            print(variables)
+            data = run_graphql_query(query, variables, token)
+            for pr in data["search"]["nodes"]:
+                # Users can be None if the user has been deleted.
+                if not pr["author"]:
+                    continue
+                author = pr["author"]["login"]
+                if author in triage_list:
+                    triage_list[author].add_authored()
+
+                if not pr["mergedBy"]:
+                    continue
+                merger = pr["mergedBy"]["login"]
+                if author == merger:
+                    continue
+                if merger not in triage_list:
+                    continue
+                triage_list[merger].add_merged()
+
+            has_next_page = data["search"]["pageInfo"]["hasNextPage"]
+            if has_next_page:
+                variables["after"] = data["search"]["pageInfo"]["endCursor"]
+        date_begin = date_end
+
+
+def main():
+    token = sys.argv[1]
+    gh = github.Github(login_or_token=token)
+    org = gh.get_organization("llvm")
+    repo = org.get_repo("llvm-project")
+    team = org.get_team_by_slug("llvm-committers")
+    one_year_ago = datetime.datetime.now() - datetime.timedelta(days=365)
+    triage_list = {}
+    for member in team.get_members():
+        triage_list[member.login] = User(member.login, triage_list)
+
+    print("Start:", len(triage_list), "triagers")
+    # Step 0 Check if users have requested commit access in the last year.
+    for user in check_manual_requests(one_year_ago, token):
+        if user in triage_list:
+            print(user, "requested commit access in the last year.")
+            del triage_list[user]
+    print("After Request Check:", len(triage_list), "triagers")
+
+    # Step 1 count all PRs authored or merged
+    count_prs(triage_list, one_year_ago, token)
+
+    print("After PRs:", len(triage_list), "triagers")
+
+    if len(triage_list) == 0:
+        sys.exit(0)
+
+    # Step 2 check for reviews
+    for user in list(triage_list.keys()):
+        review_count = get_review_count(user, one_year_ago, token)
+        triage_list[user].add_reviewed(review_count)
+
+    print("After Reviews:", len(triage_list), "triagers")
+
+    if len(triage_list) == 0:
+        sys.exit(0)
+
+    # Step 3 check for number of commits
+    for user in list(triage_list.keys()):
+        num_commits = get_num_commits(user, one_year_ago, token)
+        # Override the total number of commits to not double count commits and
+        # authored PRs.
+        triage_list[user].set_authored(num_commits)
+
+    print("After Commits:", len(triage_list), "triagers")
+
+    # Step 4 check for new committers
+    for user in list(triage_list.keys()):
+        print("Checking", user)
+        if is_new_committer(user, one_year_ago, token):
+            print("Removing new committer: ", user)
+            del triage_list[user]
+
+    print("Complete:", len(triage_list), "triagers")
+
+    with open("triagers.log", "w") as triagers_log:
+        for user in triage_list:
+            print(triage_list[user].__repr__())
+            triagers_log.write(user + "\n")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/commit-access-review.yml b/.github/workflows/commit-access-review.yml
new file mode 100644
index 000000000000000..f9195a1863deec9
--- /dev/null
+++ b/.github/workflows/commit-access-review.yml
@@ -0,0 +1,34 @@
+name: Commit Access Review
+
+on:
+  workflow_dispatch:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '0 7 1 * *'
+
+permissions:
+  contents: read
+
+jobs:
+  commit-access-review:
+    if: github.repository_owner == 'llvm'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fetch LLVM sources
+        uses: actions/checkout@v4
+      
+      - name: Install dependencies
+        run: |
+          pip install --require-hashes -r ./llvm/utils/git/requirements.txt
+      
+      - name: Run Script
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
+        run: |
+          python3 .github/workflows/commit-access-review.py $GITHUB_TOKEN
+
+      - name: Upload Triage List
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+        with:
+          name: triagers
+          path: triagers.log
diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index bfb04e7407b3804..1f2fb0a8b2b855d 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -415,6 +415,10 @@ template <> llvm::Expected<CommentInfo *> getCommentInfo(TypedefInfo *I) {
   return &I->Description.emplace_back();
 }
 
+template <> llvm::Expected<CommentInfo *> getCommentInfo(EnumValueInfo *I) {
+  return &I->Description.emplace_back();
+}
+
 template <> llvm::Expected<CommentInfo *> getCommentInfo(CommentInfo *I) {
   I->Children.emplace_back(std::make_unique<CommentInfo>());
   return I->Children.back().get();
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.cpp b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
index 7e5a11783d303a6..06f30f76e33d8ce 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
@@ -536,6 +536,8 @@ void ClangDocBitcodeWriter::emitBlock(const EnumValueInfo &I) {
   emitRecord(I.Name, ENUM_VALUE_NAME);
   emitRecord(I.Value, ENUM_VALUE_VALUE);
   emitRecord(I.ValueExpr, ENUM_VALUE_EXPR);
+  for (const auto &CI : I.Description)
+    emitBlock(CI);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const RecordInfo &I) {
diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index 21ee417da6028a8..068eec6599ead86 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -48,6 +48,12 @@ class HTMLTag {
     TAG_SPAN,
     TAG_TITLE,
     TAG_UL,
+    TAG_TABLE,
+    TAG_THEAD,
+    TAG_TBODY,
+    TAG_TR,
+    TAG_TD,
+    TAG_TH
   };
 
   HTMLTag() = default;
@@ -133,6 +139,12 @@ bool HTMLTag::isSelfClosing() const {
   case HTMLTag::TAG_SPAN:
   case HTMLTag::TAG_TITLE:
   case HTMLTag::TAG_UL:
+  case HTMLTag::TAG_TABLE:
+  case HTMLTag::TAG_THEAD:
+  case HTMLTag::TAG_TBODY:
+  case HTMLTag::TAG_TR:
+  case HTMLTag::TAG_TD:
+  case HTMLTag::TAG_TH:
     return false;
   }
   llvm_unreachable("Unhandled HTMLTag::TagType");
@@ -174,6 +186,18 @@ StringRef HTMLTag::toString() const {
     return "title";
   case HTMLTag::TAG_UL:
     return "ul";
+  case HTMLTag::TAG_TABLE:
+    return "table";
+  case HTMLTag::TAG_THEAD:
+    return "thead";
+  case HTMLTag::TAG_TBODY:
+    return "tbody";
+  case HTMLTag::TAG_TR:
+    return "tr";
+  case HTMLTag::TAG_TD:
+    return "td";
+  case HTMLTag::TAG_TH:
+    return "th";
   }
   llvm_unreachable("Unhandled HTMLTag::TagType");
 }
@@ -377,10 +401,27 @@ genEnumMembersBlock(const llvm::SmallVector<EnumValueInfo, 4> &Members) {
   if (Members.empty())
     return nullptr;
 
-  auto List = std::make_unique<TagNode>(HTMLTag::TAG_UL);
-  for (const auto &M : Members)
-    List->Children.emplace_back(
-        std::make_unique<TagNode>(HTMLTag::TAG_LI, M.Name));
+  auto List = std::make_unique<TagNode>(HTMLTag::TAG_TBODY);
+
+  for (const auto &M : Members) {
+    auto TRNode = std::make_unique<TagNode>(HTMLTag::TAG_TR);
+    TRNode->Children.emplace_back(
+        std::make_unique<TagNode>(HTMLTag::TAG_TD, M.Name));
+    // Use user supplied value if it exists, otherwise use the value
+    if (!M.ValueExpr.empty()) {
+      TRNode->Children.emplace_back(
+          std::make_unique<TagNode>(HTMLTag::TAG_TD, M.ValueExpr));
+    } else {
+      TRNode->Children.emplace_back(
+          std::make_unique<TagNode>(HTMLTag::TAG_TD, M.Value));
+    }
+    if (!M.Description.empty()) {
+      auto TD = std::make_unique<TagNode>(HTMLTag::TAG_TD);
+      TD->Children.emplace_back(genHTML(M.Description));
+      TRNode->Children.emplace_back(std::move(TD));
+    }
+    List->Children.emplace_back(std::move(TRNode));
+  }
   return List;
 }
 
@@ -624,7 +665,7 @@ static std::unique_ptr<HTMLNode> genHTML(const CommentInfo &I) {
     }
     return std::move(FullComment);
   }
- 
+
   if (I.Kind == "ParagraphComment") {
     auto ParagraphComment = std::make_unique<TagNode>(HTMLTag::TAG_P);
     for (const auto &Child : I.Children) {
@@ -658,22 +699,36 @@ static std::vector<std::unique_ptr<TagNode>>
 genHTML(const EnumInfo &I, const ClangDocContext &CDCtx) {
   std::vector<std::unique_ptr<TagNode>> Out;
   std::string EnumType = I.Scoped ? "enum class " : "enum ";
-
-  Out.emplace_back(
-      std::make_unique<TagNode>(HTMLTag::TAG_H3, EnumType + I.Name));
-  Out.back()->Attributes.emplace_back("id",
-                                      llvm::toHex(llvm::toStringRef(I.USR)));
-
-  std::unique_ptr<TagNode> Node = genEnumMembersBlock(I.Members);
-  if (Node)
-    Out.emplace_back(std::move(Node));
+  // Determine if enum members have comments attached
+  bool HasComments = std::any_of(
+      I.Members.begin(), I.Members.end(),
+      [](const EnumValueInfo &M) { return !M.Description.empty(); });
+  std::unique_ptr<TagNode> Table =
+      std::make_unique<TagNode>(HTMLTag::TAG_TABLE);
+  std::unique_ptr<TagNode> THead =
+      std::make_unique<TagNode>(HTMLTag::TAG_THEAD);
+  std::unique_ptr<TagNode> TRow = std::make_unique<TagNode>(HTMLTag::TAG_TR);
+  std::unique_ptr<TagNode> TD =
+      std::make_unique<TagNode>(HTMLTag::TAG_TH, EnumType + I.Name);
+  // Span 3 columns if enum has comments
+  TD->Attributes.emplace_back("colspan", HasComments ? "3" : "2");
+
+  Table->Attributes.emplace_back("id", llvm::toHex(llvm::toStringRef(I.USR)));
+  TRow->Children.emplace_back(std::move(TD));
+  THead->Children.emplace_back(std::move(TRow));
+  Table->Children.emplace_back(std::move(THead));
+
+  if (std::unique_ptr<TagNode> Node = genEnumMembersBlock(I.Members))
+    Table->Children.emplace_back(std::move(Node));
+
+  Out.emplace_back(std::move(Table));
 
   if (I.DefLoc) {
     if (!CDCtx.RepositoryUrl)
       Out.emplace_back(writeFileDefinition(*I.DefLoc));
     else
-      Out.emplace_back(writeFileDefinition(
-          *I.DefLoc, StringRef{*CDCtx.RepositoryUrl}));
+      Out.emplace_back(
+          writeFileDefinition(*I.DefLoc, StringRef{*CDCtx.RepositoryUrl}));
   }
 
   std::string Description;
diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 6f140910b321fb6..4da93b24c131fdf 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -221,7 +221,7 @@ void SymbolInfo::merge(SymbolInfo &&Other) {
 }
 
 NamespaceInfo::NamespaceInfo(SymbolID USR, StringRef Name, StringRef Path)
-      : Info(InfoType::IT_namespace, USR, Name, Path) {}
+    : Info(InfoType::IT_namespace, USR, Name, Path) {}
 
 void NamespaceInfo::merge(NamespaceInfo &&Other) {
   assert(mergeable(Other));
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index 0695197cf3ac959..bb0c534af7b740a 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -455,6 +455,8 @@ struct EnumValueInfo {
   // Stores the user-supplied initialization expression for this enumeration
   // constant. This will be empty for implicit enumeration values.
   SmallString<16> ValueExpr;
+
+  std::vector<CommentInfo> Description; /// Comment description of this field.
 };
 
 // TODO: Expand to allow for documenting templating.
diff --git a/clang-tools-extra/clang-doc/Serialize.cpp b/clang-tools-extra/clang-doc/Serialize.cpp
index 3b074d849e8a9cf..b9db78cf7d688fb 100644
--- a/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/clang-tools-extra/clang-doc/Serialize.cpp
@@ -394,10 +394,20 @@ static void parseEnumerators(EnumInfo &I, const EnumDecl *D) {
     std::string ValueExpr;
     if (const Expr *InitExpr = E->getInitExpr())
       ValueExpr = getSourceCode(D, InitExpr->getSourceRange());
-
     SmallString<16> ValueStr;
     E->getInitVal().toString(ValueStr);
-    I.Members.emplace_back(E->getNameAsString(), ValueStr, ValueExpr);
+    I.Members.emplace_back(E->getNameAsString(), ValueStr.str(), ValueExpr);
+    ASTContext &Context = E->getASTContext();
+    if (RawComment *Comment =
+            E->getASTContext().getRawCommentForDeclNoCache(E)) {
+      CommentInfo CInfo;
+      Comment->setAttached();
+      if (comments::FullComment *Fc = Comment->parse(Context, nullptr, E)) {
+        EnumValueInfo &Member = I.Members.back();
+        Member.Description.emplace_back();
+        parseFullComment(Fc, Member.Description.back());
+      }
+    }
   }
 }
 
@@ -566,7 +576,7 @@ static void populateMemberTypeInfo(MemberTypeInfo &I, const FieldDecl *D) {
     return;
 
   Comment->setAttached();
-  if (comments::FullComment* fc = Comment->parse(Context, nullptr, D)) {
+  if (comments::FullComment *fc = Comment->parse(Context, nullptr, D)) {
     I.Description.emplace_back();
     parseFullComment(fc, I.Description.back());
   }
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
index 6cef21f1318a2a9..cdb34aef1b0e61e 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
@@ -50,8 +50,7 @@ void UseStdFormatCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       callExpr(argumentCountAtLeast(1),
                hasArgument(0, stringLiteral(isOrdinary())),
-               callee(functionDecl(unless(cxxMethodDecl()),
-                                   matchers::matchesAnyListedName(
+               callee(functionDecl(matchers::matchesAnyListedName(
                                        StrFormatLikeFunctions))
                           .bind("func_decl")))
           .bind("strformat"),
@@ -93,7 +92,8 @@ void UseStdFormatCheck::check(const MatchFinder::MatchResult &Result) {
       diag(StrFormatCall->getBeginLoc(), "use '%0' instead of %1")
       << ReplacementFormatFunction << OldFunction->getIdentifier();
   Diag << FixItHint::CreateReplacement(
-      CharSourceRange::getTokenRange(StrFormatCall->getSourceRange()),
+      CharSourceRange::getTokenRange(StrFormatCall->getExprLoc(),
+                                     StrFormatCall->getEndLoc()),
       ReplacementFormatFunction);
   Converter.applyFixes(Diag, *Result.SourceManager);
 
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
index ff990feadc0c1d9..16f2f4b3e7d1af6 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
@@ -100,8 +100,7 @@ void UseStdPrintCheck::registerMatchers(MatchFinder *Finder) {
         unusedReturnValue(
             callExpr(argumentCountAtLeast(1),
                      hasArgument(0, stringLiteral(isOrdinary())),
-                     callee(functionDecl(unless(cxxMethodDecl()),
-                                         matchers::matchesAnyListedName(
+                     callee(functionDecl(matchers::matchesAnyListedName(
                                              PrintfLikeFunctions))
                                 .bind("func_decl")))
                 .bind("printf")),
@@ -112,8 +111,7 @@ void UseStdPrintCheck::registerMatchers(MatchFinder *Finder) {
         unusedReturnValue(
             callExpr(argumentCountAtLeast(2),
                      hasArgument(1, stringLiteral(isOrdinary())),
-                     callee(functionDecl(unless(cxxMethodDecl()),
-                                         matchers::matchesAnyListedName(
+                     callee(functionDecl(matchers::matchesAnyListedName(
                                              FprintfLikeFunctions))
                                 .bind("func_decl")))
                 .bind("fprintf")),
@@ -152,7 +150,7 @@ void UseStdPrintCheck::check(const MatchFinder::MatchResult &Result) {
       << ReplacementFunction << OldFunction->getIdentifier();
 
   Diag << FixItHint::CreateReplacement(
-      CharSourceRange::getTokenRange(PrintfCall->getBeginLoc(),
+      CharSourceRange::getTokenRange(PrintfCall->getExprLoc(),
                                      PrintfCall->getEndLoc()),
       ReplacementFunction);
   Converter.applyFixes(Diag, *Result.SourceManager);
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index a2f1504db7e880e..14440acd08b3533 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -512,8 +512,8 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
   auto Action = std::make_unique<ClangdFrontendAction>();
   const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
   if (!Action->BeginSourceFile(*Clang, MainInput)) {
-    log("BeginSourceFile() failed when building AST for {0}",
-        MainInput.getFile());
+    elog("BeginSourceFile() failed when building AST for {0}",
+         MainInput.getFile());
     return std::nullopt;
   }
   // If we saw an include guard in the preamble section of the main file,
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index b72d109b3d3938b..1b025e8f90f7ba8 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -104,9 +104,17 @@ New check aliases
 Changes in existing checks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Improved :doc:`modernize-use-std-format
+  <clang-tidy/checks/modernize/use-std-format>` check to support replacing
+  member function calls too.
+
+- Improved :doc:`modernize-use-std-print
+  <clang-tidy/checks/modernize/use-std-print>` check to support replacing
+  member function calls too.
+
 - Improved :doc:`readability-redundant-smartptr-get
   <clang-tidy/checks/readability/redundant-smartptr-get>` check to
-  remove `->`, when reduntant `get()` is removed.
+  remove `->`, when redundant `get()` is removed.
 
 Removed checks
 ^^^^^^^^^^^^^^
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
index a1599f0fc58fe68..1ec753ef090de19 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
@@ -64,8 +64,9 @@ Options
    A semicolon-separated list of (fully qualified) function names to
    replace, with the requirement that the first parameter contains the
    printf-style format string and the arguments to be formatted follow
-   immediately afterwards. The default value for this option is
-   `absl::StrFormat`.
+   immediately afterwards. Qualified member function names are supported,
+   but the replacement function name must be unqualified. The default value
+   for this option is `absl::StrFormat`.
 
 .. option:: ReplacementFormatFunction
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
index 79648a1104bca23..59bb722e2c24fc1 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
@@ -121,9 +121,10 @@ Options
    A semicolon-separated list of (fully qualified) function names to
    replace, with the requirement that the first parameter contains the
    printf-style format string and the arguments to be formatted follow
-   immediately afterwards. If neither this option nor
-   `FprintfLikeFunctions` are set then the default value for this option
-   is `printf; absl::PrintF`, otherwise it is empty.
+   immediately afterwards. Qualified member function names are supported,
+   but the replacement function name must be unqualified. If neither this
+   option nor `FprintfLikeFunctions` are set then the default value for
+   this option is `printf; absl::PrintF`, otherwise it is empty.
 
 
 .. option:: FprintfLikeFunctions
@@ -131,9 +132,11 @@ Options
    A semicolon-separated list of (fully qualified) function names to
    replace, with the requirement that the first parameter is retained, the
    second parameter contains the printf-style format string and the
-   arguments to be formatted follow immediately afterwards. If neither this
-   option nor `PrintfLikeFunctions` are set then the default value for
-   this option is `fprintf; absl::FPrintF`, otherwise it is empty.
+   arguments to be formatted follow immediately afterwards. Qualified
+   member function names are supported, but the replacement function name
+   must be unqualified. If neither this option nor `PrintfLikeFunctions`
+   are set then the default value for this option is `fprintf;
+   absl::FPrintF`, otherwise it is empty.
 
 .. option:: ReplacementPrintFunction
 
diff --git a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
index b15d428326ac12d..a5ac3760a3be2aa 100644
--- a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
+++ b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
@@ -23,7 +23,6 @@
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Casting.h"
@@ -350,6 +349,15 @@ class ASTWalker : public RecursiveASTVisitor<ASTWalker> {
            RefType::Implicit);
     return true;
   }
+
+  bool VisitCXXNewExpr(CXXNewExpr *E) {
+    report(E->getExprLoc(), E->getOperatorNew());
+    return true;
+  }
+  bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
+    report(E->getExprLoc(), E->getOperatorDelete());
+    return true;
+  }
 };
 
 } // namespace
diff --git a/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp b/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
index 282abce3246ca3d..b0a4473d4ad2b79 100644
--- a/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
+++ b/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
@@ -555,5 +555,12 @@ TEST(WalkAST, FriendDecl) {
   testWalk("void $explicit^foo();", "struct Bar { friend void ^foo(); };");
   testWalk("struct $explicit^Foo {};", "struct Bar { friend struct ^Foo; };");
 }
+
+TEST(WalkAST, OperatorNewDelete) {
+  testWalk("void* $explicit^operator new(decltype(sizeof(int)), void*);",
+           "struct Bar { void foo() { Bar b; ^new (&b) Bar; } };");
+  testWalk("struct A { static void $explicit^operator delete(void*); };",
+           "void foo() { A a; ^delete &a; }");
+}
 } // namespace
 } // namespace clang::include_cleaner
diff --git a/clang-tools-extra/test/clang-doc/enum.cpp b/clang-tools-extra/test/clang-doc/enum.cpp
index e559940a31de69a..ef768e33b456680 100644
--- a/clang-tools-extra/test/clang-doc/enum.cpp
+++ b/clang-tools-extra/test/clang-doc/enum.cpp
@@ -21,9 +21,9 @@
 enum Color {
 // MD-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
 // HTML-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-  Red, ///< Red
-  Green, ///< Green
-  Blue ///< Blue
+  Red, ///< Comment 1
+  Green, ///< Comment 2
+  Blue ///< Comment 3
 };
 
 // MD-INDEX: ## Enums
@@ -34,11 +34,16 @@ enum Color {
 // MD-INDEX: | Blue |
 // MD-INDEX: **brief** For specifying RGB colors
 
-// HTML-INDEX: <h2 id="Enums">Enums</h2>
-// HTML-INDEX: <h3 id="{{([0-9A-F]{40})}}">enum Color</h3>
-// HTML-INDEX: <li>Red</li>
-// HTML-INDEX: <li>Green</li>
-// HTML-INDEX: <li>Blue</li>
+// HTML-INDEX: <th colspan="3">enum Color</th>
+// HTML-INDEX: <td>Red</td>
+// HTML-INDEX: <td>0</td>
+// HTML-INDEX: <p> Comment 1</p>
+// HTML-INDEX: <td>Green</td>
+// HTML-INDEX: <td>1</td>
+// HTML-INDEX: <p> Comment 2</p>
+// HTML-INDEX: <td>Blue</td>
+// HTML-INDEX: <td>2</td>
+// HTML-INDEX: <p> Comment 3</p>
 
 /**
  * @brief Shape Types
@@ -46,11 +51,12 @@ enum Color {
 enum class Shapes {
 // MD-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
 // HTML-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-  /// Circle
+
+  /// Comment 1
   Circle,
-  /// Rectangle
+  /// Comment 2
   Rectangle,
-  /// Triangle
+  /// Comment 3
   Triangle
 };
 // MD-INDEX: | enum class Shapes |
@@ -60,10 +66,17 @@ enum class Shapes {
 // MD-INDEX: | Triangle |
 // MD-INDEX: **brief** Shape Types
 
-// HTML-INDEX: <h3 id="{{([0-9A-F]{40})}}">enum class Shapes</h3>
-// HTML-INDEX: <li>Circle</li>
-// HTML-INDEX: <li>Rectangle</li>
-// HTML-INDEX: <li>Triangle</li>
+// HTML-INDEX: <th colspan="3">enum class Shapes</th>
+// HTML-INDEX: <td>Circle</td>
+// HTML-INDEX: <td>0</td>
+// HTML-INDEX: <p> Comment 1</p>
+// HTML-INDEX: <td>Rectangle</td>
+// HTML-INDEX: <td>1</td>
+// HTML-INDEX: <p> Comment 2</p>
+// HTML-INDEX: <td>Triangle</td>
+// HTML-INDEX: <td>2</td>
+// HTML-INDEX: <p> Comment 3</p>
+
 
 
 class Animals {
@@ -76,18 +89,25 @@ class Animals {
       enum AnimalType {
 // MD-ANIMAL-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
 // HTML-ANIMAL-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-          Dog, /// Man's best friend
-          Cat, /// Man's other best friend
-          Iguana /// A lizard
+          Dog, ///< Man's best friend
+          Cat, ///< Man's other best friend
+          Iguana ///< A lizard
       };
 };
 
 // HTML-ANIMAL: <h1>class Animals</h1>
 // HTML-ANIMAL: <h2 id="Enums">Enums</h2>
-// HTML-ANIMAL: <h3 id="{{([0-9A-F]{40})}}">enum AnimalType</h3>
-// HTML-ANIMAL: <li>Dog</li>
-// HTML-ANIMAL: <li>Cat</li>
-// HTML-ANIMAL: <li>Iguana</li>
+// HTML-ANIMAL: <th colspan="3">enum AnimalType</th>
+// HTML-ANIMAL: <td>Dog</td>
+// HTML-ANIMAL: <td>0</td>
+// HTML-ANIMAL: <p> Man&apos;s best friend</p>
+// HTML-ANIMAL: <td>Cat</td>
+// HTML-ANIMAL: <td>1</td>
+// HTML-ANIMAL: <p> Man&apos;s other best friend</p>
+// HTML-ANIMAL: <td>Iguana</td>
+// HTML-ANIMAL: <td>2</td>
+// HTML-ANIMAL: <p> A lizard</p>
+
 
 // MD-ANIMAL: # class Animals
 // MD-ANIMAL: ## Enums
@@ -106,10 +126,11 @@ namespace Vehicles {
     enum Car {
 // MD-VEHICLES-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
 // HTML-VEHICLES-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-       Sedan, /// Sedan
-       SUV, /// SUV
-       Pickup, /// Pickup
-       Hatchback /// Hatchback
+
+       Sedan, ///< Comment 1
+       SUV, ///< Comment 2
+       Pickup, ///< Comment 3
+       Hatchback ///< Comment 4
     };
 }
 
@@ -124,9 +145,37 @@ namespace Vehicles {
 // MD-VEHICLES: **brief** specify type of car
 
 // HTML-VEHICLES: <h1>namespace Vehicles</h1>
-// HTML-VEHICLES: <h2 id="Enums">Enums</h2>
-// HTML-VEHICLES: <h3 id="{{([0-9A-F]{40})}}">enum Car</h3>
-// HTML-VEHICLES: <li>Sedan</li>
-// HTML-VEHICLES: <li>SUV</li>
-// HTML-VEHICLES: <li>Pickup</li>
-// HTML-VEHICLES: <li>Hatchback</li>
\ No newline at end of file
+// HTML-VEHICLES: <th colspan="3">enum Car</th>
+// HTML-VEHICLES: <td>Sedan</td>
+// HTML-VEHICLES: <td>0</td>
+// HTML-VEHICLES: <p> Comment 1</p>
+// HTML-VEHICLES: <td>SUV</td>
+// HTML-VEHICLES: <td>1</td>
+// HTML-VEHICLES: <p> Comment 2</p>
+// HTML-VEHICLES: <td>Pickup</td>
+// HTML-VEHICLES: <td>2</td>
+// HTML-VEHICLES: <p> Comment 3</p>
+// HTML-VEHICLES: <td>Hatchback</td>
+// HTML-VEHICLES: <td>3</td>
+// HTML-VEHICLES: <p> Comment 4</p>
+
+
+enum ColorUserSpecified {
+  RedUserSpecified = 'A',
+  GreenUserSpecified = 2,
+  BlueUserSpecified = 'C'
+};
+
+// MD-INDEX: | enum ColorUserSpecified |
+// MD-INDEX: --
+// MD-INDEX: | RedUserSpecified |
+// MD-INDEX: | GreenUserSpecified |
+// MD-INDEX: | BlueUserSpecified |
+
+// HTML-INDEX: <th colspan="2">enum ColorUserSpecified</th>
+// HTML-INDEX: <td>RedUserSpecified</td>
+// HTML-INDEX: <td>&apos;A&apos;</td>
+// HTML-INDEX: <td>GreenUserSpecified</td>
+// HTML-INDEX: <td>2</td>
+// HTML-INDEX: <td>BlueUserSpecified</td>
+// HTML-INDEX: <td>&apos;C&apos;</td>
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-member.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-member.cpp
new file mode 100644
index 000000000000000..b4e1720e0a106d0
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-member.cpp
@@ -0,0 +1,44 @@
+// RUN: %check_clang_tidy %s modernize-use-std-format %t -- \
+// RUN:   -config="{CheckOptions: \
+// RUN:             { \
+// RUN:               modernize-use-std-format.StrFormatLikeFunctions: 'MyClass::StrFormat', \
+// RUN:               modernize-use-std-format.ReplacementFormatFunction: 'format', \
+// RUN:             } \
+// RUN:            }" \
+// RUN:   -- -isystem %clang_tidy_headers
+
+#include <cstdio>
+#include <string.h>
+#include <string>
+
+struct MyClass
+{
+  template <typename S, typename... Args>
+  std::string StrFormat(const S &format, const Args&... args);
+};
+
+std::string StrFormat_simple(MyClass &myclass, MyClass *pmyclass) {
+  std::string s;
+
+  s += myclass.StrFormat("MyClass::StrFormat dot %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:8: warning: use 'format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: s += myclass.format("MyClass::StrFormat dot {}", 42);
+
+  s += pmyclass->StrFormat("MyClass::StrFormat pointer %d", 43);
+  // CHECK-MESSAGES: [[@LINE-1]]:8: warning: use 'format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: s += pmyclass->format("MyClass::StrFormat pointer {}", 43);
+
+  s += (*pmyclass).StrFormat("MyClass::StrFormat deref pointer %d", 44);
+  // CHECK-MESSAGES: [[@LINE-1]]:8: warning: use 'format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: s += (*pmyclass).format("MyClass::StrFormat deref pointer {}", 44);
+
+  return s;
+}
+
+struct MyDerivedClass : public MyClass {};
+
+std::string StrFormat_derived(MyDerivedClass &derived) {
+  return derived.StrFormat("MyDerivedClass::StrFormat dot %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: return derived.format("MyDerivedClass::StrFormat dot {}", 42);
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-print-member.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-print-member.cpp
new file mode 100644
index 000000000000000..1d1ac4b369bbaa7
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-print-member.cpp
@@ -0,0 +1,70 @@
+// RUN: %check_clang_tidy %s modernize-use-std-print %t -- \
+// RUN:   -config="{CheckOptions: \
+// RUN:             { \
+// RUN:               modernize-use-std-print.PrintfLikeFunctions: 'MyClass::printf', \
+// RUN:               modernize-use-std-print.FprintfLikeFunctions: 'MyClass::fprintf', \
+// RUN:               modernize-use-std-print.ReplacementPrintFunction: 'print', \
+// RUN:               modernize-use-std-print.ReplacementPrintlnFunction: 'println', \
+// RUN:             } \
+// RUN:            }" \
+// RUN:   -- -isystem %clang_tidy_headers
+
+#include <cstdio>
+#include <string.h>
+
+struct MyStruct {};
+
+struct MyClass
+{
+  template <typename... Args>
+  void printf(const char *fmt, Args &&...);
+  template <typename... Args>
+  int fprintf(MyStruct *param1, const char *fmt, Args &&...);
+};
+
+void printf_simple(MyClass &myclass, MyClass *pmyclass) {
+  myclass.printf("printf dot %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: myclass.print("printf dot {}", 42);
+
+  pmyclass->printf("printf pointer %d", 43);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: pmyclass->print("printf pointer {}", 43);
+
+  (*pmyclass).printf("printf deref pointer %d", 44);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: (*pmyclass).print("printf deref pointer {}", 44);
+}
+
+void printf_newline(MyClass &myclass, MyClass *pmyclass) {
+  myclass.printf("printf dot newline %c\n", 'A');
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'println' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: myclass.println("printf dot newline {}", 'A');
+
+  pmyclass->printf("printf pointer newline %c\n", 'B');
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'println' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: pmyclass->println("printf pointer newline {}", 'B');
+}
+
+void fprintf_simple(MyStruct *mystruct, MyClass &myclass, MyClass *pmyclass) {
+  myclass.fprintf(mystruct, "fprintf dot %d", 142);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'fprintf' [modernize-use-std-print]
+  // CHECK-FIXES: myclass.print(mystruct, "fprintf dot {}", 142);
+
+  pmyclass->fprintf(mystruct, "fprintf pointer %d", 143);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'fprintf' [modernize-use-std-print]
+  // CHECK-FIXES: pmyclass->print(mystruct, "fprintf pointer {}", 143);
+
+  (*pmyclass).fprintf(mystruct, "fprintf deref pointer %d", 144);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'fprintf' [modernize-use-std-print]
+  // CHECK-FIXES: (*pmyclass).print(mystruct, "fprintf deref pointer {}", 144);
+}
+
+struct MyDerivedClass : public MyClass {};
+
+void printf_derived(MyDerivedClass &derived)
+{
+  derived.printf("printf on derived class %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: derived.print("printf on derived class {}", 42);
+}
diff --git a/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp b/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
index bd031282b042abb..dfd31e657871451 100644
--- a/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
+++ b/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
@@ -92,7 +92,13 @@ TEST(HTMLGeneratorTest, emitNamespaceHTML) {
     </div>
     <h2 id="Enums">Enums</h2>
     <div>
-      <h3 id="0000000000000000000000000000000000000000">enum OneEnum</h3>
+      <table id="0000000000000000000000000000000000000000">
+        <thead>
+          <tr>
+            <th colspan="2">enum OneEnum</th>
+          </tr>
+        </thead>
+      </table>
     </div>
   </div>
   <div id="sidebar-right" class="col-xs-6 col-sm-6 col-md-2 sidebar sidebar-offcanvas-right">
@@ -214,7 +220,13 @@ TEST(HTMLGeneratorTest, emitRecordHTML) {
     </div>
     <h2 id="Enums">Enums</h2>
     <div>
-      <h3 id="0000000000000000000000000000000000000000">enum OneEnum</h3>
+      <table id="0000000000000000000000000000000000000000">
+        <thead>
+          <tr>
+            <th colspan="2">enum OneEnum</th>
+          </tr>
+        </thead>
+      </table>
     </div>
   </div>
   <div id="sidebar-right" class="col-xs-6 col-sm-6 col-md-2 sidebar sidebar-offcanvas-right">
@@ -348,10 +360,19 @@ TEST(HTMLGeneratorTest, emitEnumHTML) {
 <main>
   <div id="sidebar-left" path="" class="col-xs-6 col-sm-3 col-md-2 sidebar sidebar-offcanvas-left"></div>
   <div id="main-content" class="col-xs-12 col-sm-9 col-md-8 main-content">
-    <h3 id="0000000000000000000000000000000000000000">enum class e</h3>
-    <ul>
-      <li>X</li>
-    </ul>
+    <table id="0000000000000000000000000000000000000000">
+      <thead>
+        <tr>
+          <th colspan="2">enum class e</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>X</td>
+          <td>0</td>
+        </tr>
+      </tbody>
+    </table>
     <p>
       Defined at line 
       <a href="https://www.repository.com/test.cpp#10">10</a>
diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index c251c46a04adf43..4da99e899e7f7ce 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -319,6 +319,12 @@ def __eq__(self, other):
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def __lt__(self, other: SourceLocation) -> bool:
+        return conf.lib.clang_isBeforeInTranslationUnit(self, other)  # type: ignore [no-any-return]
+
+    def __le__(self, other: SourceLocation) -> bool:
+        return self < other or self == other
+
     def __repr__(self):
         if self.file:
             filename = self.file.name
@@ -375,26 +381,7 @@ def __contains__(self, other):
         """Useful to detect the Token/Lexer bug"""
         if not isinstance(other, SourceLocation):
             return False
-        if other.file is None and self.start.file is None:
-            pass
-        elif (
-            self.start.file.name != other.file.name
-            or other.file.name != self.end.file.name
-        ):
-            # same file name
-            return False
-        # same file, in between lines
-        if self.start.line < other.line < self.end.line:
-            return True
-        elif self.start.line == other.line:
-            # same file first line
-            if self.start.column <= other.column:
-                return True
-        elif other.line == self.end.line:
-            # same file last line
-            if other.column <= self.end.column:
-                return True
-        return False
+        return self.start <= other <= self.end
 
     def __repr__(self):
         return "<SourceRange start %r, end %r>" % (self.start, self.end)
@@ -3908,6 +3895,7 @@ def write_main_file_to_stdout(self):
     ("clang_isUnexposed", [CursorKind], bool),
     ("clang_isVirtualBase", [Cursor], bool),
     ("clang_isVolatileQualifiedType", [Type], bool),
+    ("clang_isBeforeInTranslationUnit", [SourceLocation, SourceLocation], bool),
     (
         "clang_parseTranslationUnit",
         [Index, c_interop_string, c_void_p, c_int, c_void_p, c_int, c_int],
diff --git a/clang/bindings/python/tests/cindex/test_location.py b/clang/bindings/python/tests/cindex/test_location.py
index e23677a9be3c095..27854a312e6721d 100644
--- a/clang/bindings/python/tests/cindex/test_location.py
+++ b/clang/bindings/python/tests/cindex/test_location.py
@@ -130,3 +130,23 @@ def test_is_system_location(self):
         two = get_cursor(tu, "two")
         self.assertFalse(one.location.is_in_system_header)
         self.assertTrue(two.location.is_in_system_header)
+
+    def test_operator_lt(self):
+        tu = get_tu("aaaaa")
+        t1_f = tu.get_file(tu.spelling)
+        tu2 = get_tu("aaaaa")
+
+        l_t1_12 = SourceLocation.from_position(tu, t1_f, 1, 2)
+        l_t1_13 = SourceLocation.from_position(tu, t1_f, 1, 3)
+        l_t1_14 = SourceLocation.from_position(tu, t1_f, 1, 4)
+
+        l_t2_13 = SourceLocation.from_position(tu2, tu2.get_file(tu2.spelling), 1, 3)
+
+        # In same file
+        assert l_t1_12 < l_t1_13 < l_t1_14
+        assert not l_t1_13 < l_t1_12
+
+        # In same file, different TU
+        assert l_t1_12 < l_t2_13 < l_t1_14
+        assert not l_t2_13 < l_t1_12
+        assert not l_t1_14 < l_t2_13
diff --git a/clang/bindings/python/tests/cindex/test_source_range.py b/clang/bindings/python/tests/cindex/test_source_range.py
new file mode 100644
index 000000000000000..47d8960fcafb352
--- /dev/null
+++ b/clang/bindings/python/tests/cindex/test_source_range.py
@@ -0,0 +1,85 @@
+import os
+from clang.cindex import Config
+
+if "CLANG_LIBRARY_PATH" in os.environ:
+    Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
+
+import unittest
+from clang.cindex import SourceLocation, SourceRange, TranslationUnit
+
+from .util import get_tu
+
+
+def create_range(tu, line1, column1, line2, column2):
+    return SourceRange.from_locations(
+        SourceLocation.from_position(tu, tu.get_file(tu.spelling), line1, column1),
+        SourceLocation.from_position(tu, tu.get_file(tu.spelling), line2, column2),
+    )
+
+
+class TestSourceRange(unittest.TestCase):
+    def test_contains(self):
+        tu = get_tu(
+            """aaaaa
+aaaaa
+aaaaa
+aaaaa"""
+        )
+        file = tu.get_file(tu.spelling)
+
+        l13 = SourceLocation.from_position(tu, file, 1, 3)
+        l21 = SourceLocation.from_position(tu, file, 2, 1)
+        l22 = SourceLocation.from_position(tu, file, 2, 2)
+        l23 = SourceLocation.from_position(tu, file, 2, 3)
+        l24 = SourceLocation.from_position(tu, file, 2, 4)
+        l25 = SourceLocation.from_position(tu, file, 2, 5)
+        l33 = SourceLocation.from_position(tu, file, 3, 3)
+        l31 = SourceLocation.from_position(tu, file, 3, 1)
+        r22_24 = create_range(tu, 2, 2, 2, 4)
+        r23_23 = create_range(tu, 2, 3, 2, 3)
+        r24_32 = create_range(tu, 2, 4, 3, 2)
+        r14_32 = create_range(tu, 1, 4, 3, 2)
+
+        assert l13 not in r22_24  # Line before start
+        assert l21 not in r22_24  # Column before start
+        assert l22 in r22_24  # Colum on start
+        assert l23 in r22_24  # Column in range
+        assert l24 in r22_24  # Column on end
+        assert l25 not in r22_24  # Column after end
+        assert l33 not in r22_24  # Line after end
+
+        assert l23 in r23_23  # In one-column range
+
+        assert l23 not in r24_32  # Outside range in first line
+        assert l33 not in r24_32  # Outside range in last line
+        assert l25 in r24_32  # In range in first line
+        assert l31 in r24_32  # In range in last line
+
+        assert l21 in r14_32  # In range at start of center line
+        assert l25 in r14_32  # In range at end of center line
+
+        # In range within included file
+        tu2 = TranslationUnit.from_source(
+            "main.c",
+            unsaved_files=[
+                (
+                    "main.c",
+                    """int a[] = {
+#include "numbers.inc"
+};
+""",
+                ),
+                (
+                    "./numbers.inc",
+                    """1,
+2,
+3,
+4
+                 """,
+                ),
+            ],
+        )
+
+        r_curly = create_range(tu2, 1, 11, 3, 1)
+        l_f2 = SourceLocation.from_position(tu2, tu2.get_file("./numbers.inc"), 4, 1)
+        assert l_f2 in r_curly
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 72e1bd19b6b520f..c79a635d86a6ef8 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -4187,7 +4187,8 @@ the configuration (without a prefix: ``Auto``).
 
 **IndentRequiresClause** (``Boolean``) :versionbadge:`clang-format 15` :ref:`¶ <IndentRequiresClause>`
   Indent the requires clause in a template. This only applies when
-  ``RequiresClausePosition`` is ``OwnLine``, or ``WithFollowing``.
+  ``RequiresClausePosition`` is ``OwnLine``, ``OwnLineWithBrace``,
+  or ``WithFollowing``.
 
   In clang-format 12, 13 and 14 it was named ``IndentRequires``.
 
@@ -5419,23 +5420,48 @@ the configuration (without a prefix: ``Auto``).
   Possible values:
 
   * ``RCPS_OwnLine`` (in configuration: ``OwnLine``)
-    Always put the ``requires`` clause on its own line.
+    Always put the ``requires`` clause on its own line (possibly followed by
+    a semicolon).
 
     .. code-block:: c++
 
       template <typename T>
-      requires C<T>
+        requires C<T>
       struct Foo {...
 
       template <typename T>
-      requires C<T>
+      void bar(T t)
+        requires C<T>;
+
+      template <typename T>
+        requires C<T>
       void bar(T t) {...
 
       template <typename T>
       void baz(T t)
-      requires C<T>
+        requires C<T>
       {...
 
+  * ``RCPS_OwnLineWithBrace`` (in configuration: ``OwnLineWithBrace``)
+    As with ``OwnLine``, except, unless otherwise prohibited, place a
+    following open brace (of a function definition) to follow on the same
+    line.
+
+    .. code-block:: c++
+
+      void bar(T t)
+        requires C<T> {
+        return;
+      }
+
+      void bar(T t)
+        requires C<T> {}
+
+      template <typename T>
+        requires C<T>
+      void baz(T t) {
+        ...
+
   * ``RCPS_WithPreceding`` (in configuration: ``WithPreceding``)
     Try to put the clause together with the preceding part of a declaration.
     For class templates: stick to the template declaration.
diff --git a/clang/docs/ClangFormattedStatus.rst b/clang/docs/ClangFormattedStatus.rst
index 68c13076ccbb4ec..0ee0782879ef6b4 100644
--- a/clang/docs/ClangFormattedStatus.rst
+++ b/clang/docs/ClangFormattedStatus.rst
@@ -439,7 +439,7 @@ tree in terms of conformance to :doc:`ClangFormat` as of: March 06, 2022 17:32:2
      - `2`
      - `79`
      - :part:`2%`
-   * - clang/lib/AST/Interp
+   * - clang/lib/AST/ByteCode
      - `44`
      - `18`
      - `26`
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 4679dbb68b25e10..5b50ad615d2c99f 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1505,6 +1505,7 @@ Attributes on Lambda-Expressions                                              C+
 Attributes on Structured Bindings            __cpp_structured_bindings        C++26         C++03
 Pack Indexing                                __cpp_pack_indexing              C++26         C++03
 ``= delete ("should have a reason");``       __cpp_deleted_function           C++26         C++03
+Variadic Friends                             __cpp_variadic_friend            C++26         C++03
 -------------------------------------------- -------------------------------- ------------- -------------
 Designated initializers (N494)                                                C99           C89
 Array & element qualification (N2607)                                         C23           C89
@@ -3539,7 +3540,7 @@ the debugging experience.
 ``__builtin_allow_runtime_check``
 ---------------------------------
 
-``__builtin_allow_runtime_check`` return true if the check at the current
+``__builtin_allow_runtime_check`` returns true if the check at the current
 program location should be executed. It is expected to be used to implement
 ``assert`` like checks which can be safely removed by optimizer.
 
@@ -3559,28 +3560,25 @@ program location should be executed. It is expected to be used to implement
 
 **Description**
 
-``__builtin_allow_runtime_check`` is lowered to ` ``llvm.allow.runtime.check``
+``__builtin_allow_runtime_check`` is lowered to the `llvm.allow.runtime.check
 <https://llvm.org/docs/LangRef.html#llvm-allow-runtime-check-intrinsic>`_
-builtin.
-
-The ``__builtin_allow_runtime_check()`` is expected to be used with control
-flow conditions such as in ``if`` to guard expensive runtime checks. The
-specific rules for selecting permitted checks can differ and are controlled by
-the compiler options.
-
-Flags to control checks:
-* ``-mllvm -lower-allow-check-percentile-cutoff-hot=N`` where N is PGO hotness
-cutoff in range ``[0, 999999]`` to disallow checks in hot code.
-* ``-mllvm -lower-allow-check-random-rate=P`` where P is number in range
-``[0.0, 1.0]`` representation probability of keeping a check.
-* If both flags are specified, ``-lower-allow-check-random-rate`` takes
-precedence.
-* If none is specified, ``__builtin_allow_runtime_check`` is lowered as
-``true``, allowing all checks.
-
-Parameter ``kind`` is a string literal representing a user selected kind for
-guarded check. It's unused now. It will enable kind-specific lowering in future.
-E.g. a higher hotness cutoff can be used for more expensive kind of check.
+intrinsic.
+
+The ``__builtin_allow_runtime_check()`` can be used within constrol structures
+like ``if`` to guard expensive runtime checks. The return value is determined
+by the following compiler options and may differ per call site:
+
+* ``-mllvm -lower-allow-check-percentile-cutoff-hot=N``: Disable checks in hot
+  code marked by the profile summary with a hotness cutoff in the range
+  ``[0, 999999]`` (a larger N disables more checks).
+* ``-mllvm -lower-allow-check-random-rate=P``: Keep a check with probability P,
+  a floating point number in the range ``[0.0, 1.0]``.
+* If both options are specified, a check is disabled if either condition is satisfied.
+* If neither is specified, all checks are allowed.
+
+Parameter ``kind``, currently unused, is a string literal specifying the check
+kind. Future compiler versions may use this to allow for more granular control,
+such as applying different hotness cutoffs to different check kinds.
 
 Query for this feature with ``__has_builtin(__builtin_allow_runtime_check)``.
 
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index b1864901e7bddbc..249249971dec7ca 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -73,12 +73,10 @@ C++ Specific Potentially Breaking Changes
     template <> // error: extraneous template head
     template <typename T>
     void f();
-    
+
 ABI Changes in This Version
 ---------------------------
 
-- Fixed Microsoft name mangling of placeholder, auto and decltype(auto), return types for MSVC 1920+. This change resolves incompatibilities with code compiled by MSVC 1920+ but will introduce incompatibilities with code compiled by earlier versions of Clang unless such code is built with the compiler option -fms-compatibility-version=19.14 to imitate the MSVC 1914 mangling behavior.
-
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
 
@@ -92,6 +90,9 @@ Clang Python Bindings Potentially Breaking Changes
 - Calling a property on the ``CompletionChunk`` or ``CompletionString`` class
   statically now leads to an error, instead of returning a ``CachedProperty`` object
   that is used internally. Properties are only available on instances.
+- For a single-line ``SourceRange`` and a ``SourceLocation`` in the same line,
+  but after the end of the ``SourceRange``, ``SourceRange.__contains__``
+  used to incorrectly return ``True``. (#GH22617), (#GH52827)
 
 What's New in Clang |release|?
 ==============================
@@ -128,6 +129,8 @@ C++2c Feature Support
 - Add ``__builtin_is_virtual_base_of`` intrinsic, which supports
   `P2985R0 A type trait for detecting virtual base classes <https://wg21.link/p2985r0>`_
 
+- Implemented `P2893R3 Variadic Friends <https://wg21.link/P2893>`_
+
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -158,6 +161,10 @@ C23 Feature Support
 Non-comprehensive list of changes in this release
 -------------------------------------------------
 
+- The floating point comparison builtins (``__builtin_isgreater``,
+  ``__builtin_isgreaterequal``, ``__builtin_isless``, etc.) and
+  ``__builtin_signbit`` can now be used in constant expressions.
+
 New Compiler Flags
 ------------------
 
@@ -167,6 +174,8 @@ Deprecated Compiler Flags
 Modified Compiler Flags
 -----------------------
 
+- The compiler flag `-fbracket-depth` default value is increased from 256 to 2048.
+
 Removed Compiler Flags
 -------------------------
 
@@ -186,6 +195,9 @@ Attribute Changes in Clang
 - The ``hybrid_patchable`` attribute is now supported on ARM64EC targets. It can be used to specify
   that a function requires an additional x86-64 thunk, which may be patched at runtime.
 
+- ``[[clang::lifetimebound]]`` is now explicitly disallowed on explicit object member functions
+  where they were previously silently ignored.
+
 Improvements to Clang's diagnostics
 -----------------------------------
 
@@ -211,6 +223,8 @@ Improvements to Clang's diagnostics
 
 - Clang now diagnoses the use of ``main`` in an ``extern`` context as invalid according to [basic.start.main] p3. Fixes #GH101512.
 
+- Clang now diagnoses when the result of a [[nodiscard]] function is discarded after being cast in C. Fixes #GH104391.
+
 Improvements to Clang's time-trace
 ----------------------------------
 
@@ -256,10 +270,18 @@ Bug Fixes to C++ Support
   specialization of a conversion function template.
 - Correctly diagnose attempts to use a concept name in its own definition;
   A concept name is introduced to its scope sooner to match the C++ standard. (#GH55875)
+- Properly reject defaulted relational operators with invalid types for explicit object parameters,
+  e.g., ``bool operator==(this int, const Foo&)`` (#GH100329), and rvalue reference parameters.
+- Properly reject defaulted copy/move assignment operators that have a non-reference explicit object parameter.
+- Clang now properly handles the order of attributes in `extern` blocks. (#GH101990).
+- Fixed an assertion failure by preventing null explicit object arguments from being deduced. (#GH102025).
+- Correctly check constraints of explicit instantiations of member functions. (#GH46029)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Fixed a crash that occurred when dividing by zero in complex integer division. (#GH55390).
+
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -313,6 +335,12 @@ Android Support
 Windows Support
 ^^^^^^^^^^^^^^^
 
+- Clang no longer allows references inside a union when emulating MSVC 1900+ even if `fms-extensions` is enabled.
+  Starting with VS2015, MSVC 1900, this Microsoft extension is no longer allowed and always results in an error.
+  Clang now follows the MSVC behavior in this scenario.
+  When `-fms-compatibility-version=18.00` or prior is set on the command line this Microsoft extension is still
+  allowed as VS2013 and prior allow it.
+
 LoongArch Support
 ^^^^^^^^^^^^^^^^^
 
@@ -359,6 +387,8 @@ clang-format
 
 libclang
 --------
+- Add ``clang_isBeforeInTranslationUnit``. Given two source locations, it determines
+  whether the first one comes strictly before the second in the source code.
 
 Static Analyzer
 ---------------
@@ -394,36 +424,6 @@ Moved checkers
 Sanitizers
 ----------
 
-- Added the ``-fsanitize-overflow-pattern-exclusion=`` flag which can be used
-  to disable specific overflow-dependent code patterns. The supported patterns
-  are: ``add-overflow-test``, ``negated-unsigned-const``, and
-  ``post-decr-while``. The sanitizer instrumentation can be toggled off for all
-  available patterns by specifying ``all``. Conversely, you can disable all
-  exclusions with ``none``.
-
-  .. code-block:: c++
-
-     /// specified with ``-fsanitize-overflow-pattern-exclusion=add-overflow-test``
-     int common_overflow_check_pattern(unsigned base, unsigned offset) {
-       if (base + offset < base) { /* ... */ } // The pattern of `a + b < a`, and other re-orderings, won't be instrumented
-     }
-
-     /// specified with ``-fsanitize-overflow-pattern-exclusion=negated-unsigned-const``
-     void negation_overflow() {
-       unsigned long foo = -1UL; // No longer causes a negation overflow warning
-       unsigned long bar = -2UL; // and so on...
-     }
-
-     /// specified with ``-fsanitize-overflow-pattern-exclusion=post-decr-while``
-     void while_post_decrement() {
-       unsigned char count = 16;
-       while (count--) { /* ... */} // No longer causes unsigned-integer-overflow sanitizer to trip
-     }
-
-  Many existing projects have a large amount of these code patterns present.
-  This new flag should allow those projects to enable integer sanitizers with
-  less noise.
-
 Python Binding Changes
 ----------------------
 - Fixed an issue that led to crashes when calling ``Type.get_exception_specification_kind``.
diff --git a/clang/docs/UndefinedBehaviorSanitizer.rst b/clang/docs/UndefinedBehaviorSanitizer.rst
index 9f3d980eefbea77..531d56e313826c7 100644
--- a/clang/docs/UndefinedBehaviorSanitizer.rst
+++ b/clang/docs/UndefinedBehaviorSanitizer.rst
@@ -293,48 +293,6 @@ To silence reports from unsigned integer overflow, you can set
 ``-fsanitize-recover=unsigned-integer-overflow``, is particularly useful for
 providing fuzzing signal without blowing up logs.
 
-Disabling instrumentation for common overflow patterns
-------------------------------------------------------
-
-There are certain overflow-dependent or overflow-prone code patterns which
-produce a lot of noise for integer overflow/truncation sanitizers. Negated
-unsigned constants, post-decrements in a while loop condition and simple
-overflow checks are accepted and pervasive code patterns. However, the signal
-received from sanitizers instrumenting these code patterns may be too noisy for
-some projects. To disable instrumentation for these common patterns one should
-use ``-fsanitize-overflow-pattern-exclusion=``.
-
-Currently, this option supports three overflow-dependent code idioms:
-
-``negated-unsigned-const``
-
-.. code-block:: c++
-
-    /// -fsanitize-overflow-pattern-exclusion=negated-unsigned-const
-    unsigned long foo = -1UL; // No longer causes a negation overflow warning
-    unsigned long bar = -2UL; // and so on...
-
-``post-decr-while``
-
-.. code-block:: c++
-
-    /// -fsanitize-overflow-pattern-exclusion=post-decr-while
-    unsigned char count = 16;
-    while (count--) { /* ... */ } // No longer causes unsigned-integer-overflow sanitizer to trip
-
-``add-overflow-test``
-
-.. code-block:: c++
-
-    /// -fsanitize-overflow-pattern-exclusion=add-overflow-test
-    if (base + offset < base) { /* ... */ } // The pattern of `a + b < a`, and other re-orderings,
-                                            // won't be instrumented (same for signed types)
-
-You can enable all exclusions with
-``-fsanitize-overflow-pattern-exclusion=all`` or disable all exclusions with
-``-fsanitize-overflow-pattern-exclusion=none``. Specifying ``none`` has
-precedence over other values.
-
 Issue Suppression
 =================
 
diff --git a/clang/docs/tools/clang-formatted-files.txt b/clang/docs/tools/clang-formatted-files.txt
index 62871133a680753..fc07357986d989f 100644
--- a/clang/docs/tools/clang-formatted-files.txt
+++ b/clang/docs/tools/clang-formatted-files.txt
@@ -322,24 +322,24 @@ clang/lib/APINotes/APINotesTypes.cpp
 clang/lib/APINotes/APINotesYAMLCompiler.cpp
 clang/lib/AST/DataCollection.cpp
 clang/lib/AST/Linkage.h
-clang/lib/AST/Interp/ByteCodeGenError.cpp
-clang/lib/AST/Interp/ByteCodeGenError.h
-clang/lib/AST/Interp/Context.cpp
-clang/lib/AST/Interp/Context.h
-clang/lib/AST/Interp/Descriptor.cpp
-clang/lib/AST/Interp/Disasm.cpp
-clang/lib/AST/Interp/EvalEmitter.h
-clang/lib/AST/Interp/Frame.cpp
-clang/lib/AST/Interp/Frame.h
-clang/lib/AST/Interp/InterpState.h
-clang/lib/AST/Interp/Opcode.h
-clang/lib/AST/Interp/Pointer.cpp
-clang/lib/AST/Interp/PrimType.cpp
-clang/lib/AST/Interp/Record.h
-clang/lib/AST/Interp/Source.cpp
-clang/lib/AST/Interp/Source.h
-clang/lib/AST/Interp/State.cpp
-clang/lib/AST/Interp/State.h
+clang/lib/AST/ByteCode/ByteCodeGenError.cpp
+clang/lib/AST/ByteCode/ByteCodeGenError.h
+clang/lib/AST/ByteCode/Context.cpp
+clang/lib/AST/ByteCode/Context.h
+clang/lib/AST/ByteCode/Descriptor.cpp
+clang/lib/AST/ByteCode/Disasm.cpp
+clang/lib/AST/ByteCode/EvalEmitter.h
+clang/lib/AST/ByteCode/Frame.cpp
+clang/lib/AST/ByteCode/Frame.h
+clang/lib/AST/ByteCode/InterpState.h
+clang/lib/AST/ByteCode/Opcode.h
+clang/lib/AST/ByteCode/Pointer.cpp
+clang/lib/AST/ByteCode/PrimType.cpp
+clang/lib/AST/ByteCode/Record.h
+clang/lib/AST/ByteCode/Source.cpp
+clang/lib/AST/ByteCode/Source.h
+clang/lib/AST/ByteCode/State.cpp
+clang/lib/AST/ByteCode/State.h
 clang/lib/ASTMatchers/GtestMatchers.cpp
 clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
 clang/lib/Basic/Attributes.cpp
diff --git a/clang/include/clang-c/CXSourceLocation.h b/clang/include/clang-c/CXSourceLocation.h
index dcb13ba273173fd..421802151d02ab8 100644
--- a/clang/include/clang-c/CXSourceLocation.h
+++ b/clang/include/clang-c/CXSourceLocation.h
@@ -74,6 +74,16 @@ CINDEX_LINKAGE CXSourceLocation clang_getNullLocation(void);
 CINDEX_LINKAGE unsigned clang_equalLocations(CXSourceLocation loc1,
                                              CXSourceLocation loc2);
 
+/**
+ * Determine for two source locations if the first comes
+ * strictly before the second one in the source code.
+ *
+ * \returns non-zero if the first source location comes
+ * strictly before the second one, zero otherwise.
+ */
+CINDEX_LINKAGE unsigned clang_isBeforeInTranslationUnit(CXSourceLocation loc1,
+                                                        CXSourceLocation loc2);
+
 /**
  * Returns non-zero if the given source location is in a system header.
  */
diff --git a/clang/include/clang/APINotes/APINotesReader.h b/clang/include/clang/APINotes/APINotesReader.h
index 03657352c49a59e..baf63340640242a 100644
--- a/clang/include/clang/APINotes/APINotesReader.h
+++ b/clang/include/clang/APINotes/APINotesReader.h
@@ -141,6 +141,13 @@ class APINotesReader {
                                                  ObjCSelectorRef Selector,
                                                  bool IsInstanceMethod);
 
+  /// Look for information regarding the given field of a C struct.
+  ///
+  /// \param Name The name of the field.
+  ///
+  /// \returns information about the field, if known.
+  VersionedInfo<FieldInfo> lookupField(ContextID CtxID, llvm::StringRef Name);
+
   /// Look for information regarding the given C++ method in the given C++ tag
   /// context.
   ///
diff --git a/clang/include/clang/APINotes/APINotesWriter.h b/clang/include/clang/APINotes/APINotesWriter.h
index e0fe5eacef72582..3cc16c3d959faf6 100644
--- a/clang/include/clang/APINotes/APINotesWriter.h
+++ b/clang/include/clang/APINotes/APINotesWriter.h
@@ -86,6 +86,14 @@ class APINotesWriter {
   void addCXXMethod(ContextID CtxID, llvm::StringRef Name,
                     const CXXMethodInfo &Info, llvm::VersionTuple SwiftVersion);
 
+  /// Add information about a specific C record field.
+  ///
+  /// \param CtxID The context in which this field resides, i.e. a C/C++ tag.
+  /// \param Name The name of the field.
+  /// \param Info Information about this field.
+  void addField(ContextID CtxID, llvm::StringRef Name, const FieldInfo &Info,
+                llvm::VersionTuple SwiftVersion);
+
   /// Add information about a global variable.
   ///
   /// \param Name The name of this global variable.
diff --git a/clang/include/clang/APINotes/Types.h b/clang/include/clang/APINotes/Types.h
index f972d0cf26640d3..89889910d1a0730 100644
--- a/clang/include/clang/APINotes/Types.h
+++ b/clang/include/clang/APINotes/Types.h
@@ -656,6 +656,12 @@ class GlobalFunctionInfo : public FunctionInfo {
   GlobalFunctionInfo() {}
 };
 
+/// Describes API notes data for a C/C++ record field.
+class FieldInfo : public VariableInfo {
+public:
+  FieldInfo() {}
+};
+
 /// Describes API notes data for a C++ method.
 class CXXMethodInfo : public FunctionInfo {
 public:
diff --git a/clang/include/clang/AST/Attr.h b/clang/include/clang/AST/Attr.h
index 8e9b7ad8b468264..bd1851a26ce2e18 100644
--- a/clang/include/clang/AST/Attr.h
+++ b/clang/include/clang/AST/Attr.h
@@ -372,7 +372,7 @@ class ParamIdx {
 static_assert(sizeof(ParamIdx) == sizeof(ParamIdx::SerialType),
               "ParamIdx does not fit its serialization type");
 
-#include "clang/AST/Attrs.inc"
+#include "clang/AST/Attrs.inc" // IWYU pragma: export
 
 inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
                                              const Attr *At) {
diff --git a/clang/include/clang/AST/Availability.h b/clang/include/clang/AST/Availability.h
index 26ae622e5b4496f..60ca1383f0a44ea 100644
--- a/clang/include/clang/AST/Availability.h
+++ b/clang/include/clang/AST/Availability.h
@@ -97,6 +97,10 @@ struct AvailabilityInfo {
     return UnconditionallyUnavailable;
   }
 
+  /// Augments the existing information with additional constraints provided by
+  /// \c Other.
+  void mergeWith(AvailabilityInfo Other);
+
   AvailabilityInfo(StringRef Domain, VersionTuple I, VersionTuple D,
                    VersionTuple O, bool U, bool UD, bool UU)
       : Domain(Domain), Introduced(I), Deprecated(D), Obsoleted(O),
diff --git a/clang/include/clang/AST/DeclFriend.h b/clang/include/clang/AST/DeclFriend.h
index 9789282f351a55c..095f14a81fd5747 100644
--- a/clang/include/clang/AST/DeclFriend.h
+++ b/clang/include/clang/AST/DeclFriend.h
@@ -70,6 +70,9 @@ class FriendDecl final
   // Location of the 'friend' specifier.
   SourceLocation FriendLoc;
 
+  // Location of the '...', if present.
+  SourceLocation EllipsisLoc;
+
   /// True if this 'friend' declaration is unsupported.  Eventually we
   /// will support every possible friend declaration, but for now we
   /// silently ignore some and set this flag to authorize all access.
@@ -82,10 +85,11 @@ class FriendDecl final
   unsigned NumTPLists : 31;
 
   FriendDecl(DeclContext *DC, SourceLocation L, FriendUnion Friend,
-             SourceLocation FriendL,
+             SourceLocation FriendL, SourceLocation EllipsisLoc,
              ArrayRef<TemplateParameterList *> FriendTypeTPLists)
       : Decl(Decl::Friend, DC, L), Friend(Friend), FriendLoc(FriendL),
-        UnsupportedFriend(false), NumTPLists(FriendTypeTPLists.size()) {
+        EllipsisLoc(EllipsisLoc), UnsupportedFriend(false),
+        NumTPLists(FriendTypeTPLists.size()) {
     for (unsigned i = 0; i < NumTPLists; ++i)
       getTrailingObjects<TemplateParameterList *>()[i] = FriendTypeTPLists[i];
   }
@@ -110,7 +114,7 @@ class FriendDecl final
 
   static FriendDecl *
   Create(ASTContext &C, DeclContext *DC, SourceLocation L, FriendUnion Friend_,
-         SourceLocation FriendL,
+         SourceLocation FriendL, SourceLocation EllipsisLoc = {},
          ArrayRef<TemplateParameterList *> FriendTypeTPLists = std::nullopt);
   static FriendDecl *CreateDeserialized(ASTContext &C, GlobalDeclID ID,
                                         unsigned FriendTypeNumTPLists);
@@ -143,8 +147,24 @@ class FriendDecl final
     return FriendLoc;
   }
 
+  /// Retrieves the location of the '...', if present.
+  SourceLocation getEllipsisLoc() const { return EllipsisLoc; }
+
   /// Retrieves the source range for the friend declaration.
   SourceRange getSourceRange() const override LLVM_READONLY {
+    if (TypeSourceInfo *TInfo = getFriendType()) {
+      SourceLocation StartL =
+          (NumTPLists == 0) ? getFriendLoc()
+                            : getTrailingObjects<TemplateParameterList *>()[0]
+                                  ->getTemplateLoc();
+      SourceLocation EndL = isPackExpansion() ? getEllipsisLoc()
+                                              : TInfo->getTypeLoc().getEndLoc();
+      return SourceRange(StartL, EndL);
+    }
+
+    if (isPackExpansion())
+      return SourceRange(getFriendLoc(), getEllipsisLoc());
+
     if (NamedDecl *ND = getFriendDecl()) {
       if (const auto *FD = dyn_cast<FunctionDecl>(ND))
         return FD->getSourceRange();
@@ -158,15 +178,8 @@ class FriendDecl final
       }
       return SourceRange(getFriendLoc(), ND->getEndLoc());
     }
-    else if (TypeSourceInfo *TInfo = getFriendType()) {
-      SourceLocation StartL =
-          (NumTPLists == 0) ? getFriendLoc()
-                            : getTrailingObjects<TemplateParameterList *>()[0]
-                                  ->getTemplateLoc();
-      return SourceRange(StartL, TInfo->getTypeLoc().getEndLoc());
-    }
-    else
-      return SourceRange(getFriendLoc(), getLocation());
+
+    return SourceRange(getFriendLoc(), getLocation());
   }
 
   /// Determines if this friend kind is unsupported.
@@ -177,6 +190,8 @@ class FriendDecl final
     UnsupportedFriend = Unsupported;
   }
 
+  bool isPackExpansion() const { return EllipsisLoc.isValid(); }
+
   // Implement isa/cast/dyncast/etc.
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K == Decl::Friend; }
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index f5863524723a2ec..5b813bfc2faf908 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -4043,15 +4043,6 @@ class BinaryOperator : public Expr {
   void setHasStoredFPFeatures(bool B) { BinaryOperatorBits.HasFPFeatures = B; }
   bool hasStoredFPFeatures() const { return BinaryOperatorBits.HasFPFeatures; }
 
-  /// Set and get the bit that informs arithmetic overflow sanitizers whether
-  /// or not they should exclude certain BinaryOperators from instrumentation
-  void setExcludedOverflowPattern(bool B) {
-    BinaryOperatorBits.ExcludedOverflowPattern = B;
-  }
-  bool hasExcludedOverflowPattern() const {
-    return BinaryOperatorBits.ExcludedOverflowPattern;
-  }
-
   /// Get FPFeatures from trailing storage
   FPOptionsOverride getStoredFPFeatures() const {
     assert(hasStoredFPFeatures());
diff --git a/clang/include/clang/AST/Stmt.h b/clang/include/clang/AST/Stmt.h
index f1a2aac0a8b2f8f..bbd7634bcc3bfb3 100644
--- a/clang/include/clang/AST/Stmt.h
+++ b/clang/include/clang/AST/Stmt.h
@@ -650,11 +650,6 @@ class alignas(void *) Stmt {
     LLVM_PREFERRED_TYPE(bool)
     unsigned HasFPFeatures : 1;
 
-    /// Whether or not this BinaryOperator should be excluded from integer
-    /// overflow sanitization.
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned ExcludedOverflowPattern : 1;
-
     SourceLocation OpLoc;
   };
 
diff --git a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
index f9c923a36ad2296..ec4d041254877fd 100644
--- a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
+++ b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
@@ -139,6 +139,9 @@ struct ReferencedDecls {
   /// All variables with static storage duration, notably including static
   /// member variables and static variables declared within a function.
   llvm::DenseSet<const VarDecl *> Globals;
+  /// Local variables, not including parameters or static variables declared
+  /// within a function.
+  llvm::DenseSet<const VarDecl *> Locals;
   /// Free functions and member functions which are referenced (but not
   /// necessarily called).
   llvm::DenseSet<const FunctionDecl *> Functions;
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 0a874d8638df435..036366cdadf4aa2 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -533,42 +533,42 @@ def BuiltinComplex : Builtin {
 def IsGreater : Builtin {
   let Spellings = ["__builtin_isgreater"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsGreaterEqual : Builtin {
   let Spellings = ["__builtin_isgreaterequal"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsLess : Builtin {
   let Spellings = ["__builtin_isless"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsLessEqual : Builtin {
   let Spellings = ["__builtin_islessequal"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsLessGreater : Builtin {
   let Spellings = ["__builtin_islessgreater"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsUnordered : Builtin {
   let Spellings = ["__builtin_isunordered"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
@@ -646,19 +646,21 @@ def IsFPClass : Builtin {
 def Signbit : Builtin {
   let Spellings = ["__builtin_signbit"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def SignbitF : Builtin {
   let Spellings = ["__builtin_signbitf"];
-  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const];
+  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
+                    Constexpr];
   let Prototype = "int(float)";
 }
 
 def SignbitL : Builtin {
   let Spellings = ["__builtin_signbitl"];
-  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const];
+  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
+                    Constexpr];
   let Prototype = "int(long double)";
 }
 
diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def
index 504314d8d96e91e..c11970c279c4bba 100644
--- a/clang/include/clang/Basic/BuiltinsNVPTX.def
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.def
@@ -584,6 +584,21 @@ TARGET_BUILTIN(__nvvm_f2bf16_rz_relu, "yf", "", AND(SM_80,PTX70))
 
 TARGET_BUILTIN(__nvvm_f2tf32_rna, "ZUif", "", AND(SM_80,PTX70))
 
+TARGET_BUILTIN(__nvvm_ff_to_e4m3x2_rn, "sff", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_ff_to_e4m3x2_rn_relu, "sff", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_ff_to_e5m2x2_rn, "sff", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_ff_to_e5m2x2_rn_relu, "sff", "", AND(SM_89,PTX81))
+
+TARGET_BUILTIN(__nvvm_f16x2_to_e4m3x2_rn, "sV2h", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_f16x2_to_e4m3x2_rn_relu, "sV2h", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_f16x2_to_e5m2x2_rn, "sV2h", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_f16x2_to_e5m2x2_rn_relu, "sV2h", "", AND(SM_89,PTX81))
+
+TARGET_BUILTIN(__nvvm_e4m3x2_to_f16x2_rn, "V2hs", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_e4m3x2_to_f16x2_rn_relu, "V2hs", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_e5m2x2_to_f16x2_rn, "V2hs", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_e5m2x2_to_f16x2_rn_relu, "V2hs", "", AND(SM_89,PTX81))
+
 // Bitcast
 
 BUILTIN(__nvvm_bitcast_f2i, "if", "")
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 5728ab8d86702be..09e892d6d4defea 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -37,8 +37,6 @@ VALUE_CODEGENOPT(Name, Bits, Default)
 
 CODEGENOPT(DisableIntegratedAS, 1, 0) ///< -no-integrated-as
 CODEGENOPT(Crel, 1, 0) ///< -Wa,--crel
-CODEGENOPT(RelaxELFRelocations, 1, 1) ///< -Wa,-mrelax-relocations={yes,no}
-CODEGENOPT(SSE2AVX          , 1, 0)   ///< -msse2avx
 CODEGENOPT(AsmVerbose        , 1, 0) ///< -dA, -fverbose-asm.
 CODEGENOPT(PreserveAsmComments, 1, 1) ///< -dA, -fno-preserve-as-comments.
 CODEGENOPT(AssumeSaneOperatorNew , 1, 1) ///< implicit __attribute__((malloc)) operator new
@@ -194,6 +192,8 @@ CODEGENOPT(HIPSaveKernelArgName, 1, 0) ///< Set when -fhip-kernel-arg-name is en
 CODEGENOPT(UniqueInternalLinkageNames, 1, 0) ///< Internal Linkage symbols get unique names.
 CODEGENOPT(SplitMachineFunctions, 1, 0) ///< Split machine functions using profile information.
 CODEGENOPT(PPCUseFullRegisterNames, 1, 0) ///< Print full register names in assembly
+CODEGENOPT(X86RelaxRelocations, 1, 1) ///< -Wa,-mrelax-relocations={yes,no}
+CODEGENOPT(X86Sse2Avx        , 1, 0)   ///< -Wa,-msse2avx
 
 /// When false, this attempts to generate code as if the result of an
 /// overflowing conversion matches the overflowing behavior of a target's native
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 92a602829933ced..ba90742fbdaabcf 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -604,9 +604,6 @@ def warn_drv_unsupported_gpopt : Warning<
   "ignoring '-mgpopt' option as it cannot be used with %select{|the implicit"
   " usage of }0-mabicalls">,
   InGroup<UnsupportedGPOpt>;
-def warn_drv_unsupported_sdata : Warning<
-  "ignoring '-msmall-data-limit=' with -mcmodel=large for -fpic or RV64">,
-  InGroup<OptionIgnored>;
 def warn_drv_unsupported_longcalls : Warning<
   "ignoring '-mlong-calls' option as it is not currently supported with "
   "%select{|the implicit usage of }0-mabicalls">,
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 12aab09f285567f..62a97b36737e72e 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -965,6 +965,12 @@ def warn_cxx23_delete_with_message : Warning<
   "'= delete' with a message is incompatible with C++ standards before C++2c">,
   DefaultIgnore, InGroup<CXXPre26Compat>;
 
+def ext_variadic_friends : ExtWarn<
+  "variadic 'friend' declarations are a C++2c extension">, InGroup<CXX26>;
+def warn_cxx23_variadic_friends : Warning<
+  "variadic 'friend' declarations are incompatible with C++ standards before C++2c">,
+  DefaultIgnore, InGroup<CXXPre26Compat>;
+
 // C++11 default member initialization
 def ext_nonstatic_member_init : ExtWarn<
   "default member initializer for non-static data member is a C++11 "
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index da2f939067bfab8..4b6aadd635786a9 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -1741,6 +1741,10 @@ def ext_friend_tag_redecl_outside_namespace : ExtWarn<
   "enclosing namespace is a Microsoft extension; add a nested name specifier">,
   InGroup<MicrosoftUnqualifiedFriend>;
 def err_pure_friend : Error<"friend declaration cannot have a pure-specifier">;
+def err_friend_template_decl_multiple_specifiers: Error<
+  "a friend declaration that befriends a template must contain exactly one type-specifier">;
+def friend_template_decl_malformed_pack_expansion : Error<
+  "friend declaration expands pack %0 that is declared it its own template parameter list">;
 
 def err_invalid_base_in_interface : Error<
   "interface type cannot inherit from "
@@ -5663,6 +5667,8 @@ def err_explicit_instantiation_internal_linkage : Error<
 def err_explicit_instantiation_not_known : Error<
   "explicit instantiation of %0 does not refer to a function template, "
   "variable template, member function, member class, or static data member">;
+def err_explicit_instantiation_no_candidate : Error<
+    "no viable candidate for explicit instantiation of %0">;
 def note_explicit_instantiation_here : Note<
   "explicit instantiation refers here">;
 def err_explicit_instantiation_data_member_not_instantiated : Error<
@@ -9752,7 +9758,7 @@ def err_defaulted_special_member_quals : Error<
   "have 'const'%select{, 'constexpr'|}1 or 'volatile' qualifiers">;
 def err_defaulted_special_member_explicit_object_mismatch : Error<
   "the type of the explicit object parameter of an explicitly-defaulted "
-  "%select{copy|move}0 assignment operator should match the type of the class %1">;
+  "%select{copy|move}0 assignment operator should be reference to %1">;
 def err_defaulted_special_member_volatile_param : Error<
   "the parameter for an explicitly-defaulted %sub{select_special_member_kind}0 "
   "may not be volatile">;
@@ -10062,8 +10068,9 @@ def warn_null_ret : Warning<
   InGroup<NonNull>;
 
 def err_lifetimebound_no_object_param : Error<
-  "'lifetimebound' attribute cannot be applied; %select{static |non-}0member "
-  "function has no implicit object parameter">;
+  "'lifetimebound' attribute cannot be applied; "
+  "%select{non-|static |explicit object }0"
+  "member function has no implicit object parameter">;
 def err_lifetimebound_ctor_dtor : Error<
   "'lifetimebound' attribute cannot be applied to a "
   "%select{constructor|destructor}0">;
@@ -11160,6 +11167,8 @@ def err_omp_loop_diff_cxx : Error<
   "upper and lower loop bounds">;
 def err_omp_loop_cannot_use_stmt : Error<
   "'%0' statement cannot be used in OpenMP for loop">;
+def err_omp_loop_bad_collapse_var : Error<
+  "cannot use variable %1 in collapsed imperfectly-nested loop %select{init|condition|increment}0 statement">;
 def err_omp_simd_region_cannot_use_stmt : Error<
   "'%0' statement cannot be used in OpenMP simd region">;
 def warn_omp_loop_64_bit_var : Warning<
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 2e9f2c552aad8a2..d454a7ff2f8cf49 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -406,8 +406,6 @@ VALUE_LANGOPT(TrivialAutoVarInitMaxSize, 32, 0,
              "stop trivial automatic variable initialization if var size exceeds the specified size (in bytes). Must be greater than 0.")
 ENUM_LANGOPT(SignedOverflowBehavior, SignedOverflowBehaviorTy, 2, SOB_Undefined,
              "signed integer overflow handling")
-LANGOPT(IgnoreNegationOverflow, 1, 0, "ignore overflow caused by negation")
-LANGOPT(SanitizeOverflowIdioms, 1, 1, "enable instrumentation for common overflow idioms")
 ENUM_LANGOPT(ThreadModel  , ThreadModelKind, 2, ThreadModelKind::POSIX, "Thread Model")
 
 BENIGN_LANGOPT(ArrowDepth, 32, 256,
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index eb4cb4b5a7e93fa..91f1c2f2e6239ee 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -367,21 +367,6 @@ class LangOptionsBase {
     PerThread,
   };
 
-  /// Exclude certain code patterns from being instrumented by arithmetic
-  /// overflow sanitizers
-  enum OverflowPatternExclusionKind {
-    /// Don't exclude any overflow patterns from sanitizers
-    None = 1 << 0,
-    /// Exclude all overflow patterns (below)
-    All = 1 << 1,
-    /// if (a + b < a)
-    AddOverflowTest = 1 << 2,
-    /// -1UL
-    NegUnsignedConst = 1 << 3,
-    /// while (count--)
-    PostDecrInWhile = 1 << 4,
-  };
-
   enum class DefaultVisiblityExportMapping {
     None,
     /// map only explicit default visibilities to exported
@@ -570,11 +555,6 @@ class LangOptions : public LangOptionsBase {
   /// The default stream kind used for HIP kernel launching.
   GPUDefaultStreamKind GPUDefaultStream;
 
-  /// Which overflow patterns should be excluded from sanitizer instrumentation
-  unsigned OverflowPatternExclusionMask = 0;
-
-  std::vector<std::string> OverflowPatternExclusionValues;
-
   /// The seed used by the randomize structure layout feature.
   std::string RandstructSeed;
 
@@ -650,14 +630,6 @@ class LangOptions : public LangOptionsBase {
     return MSCompatibilityVersion >= MajorVersion * 100000U;
   }
 
-  bool isOverflowPatternExcluded(OverflowPatternExclusionKind Kind) const {
-    if (OverflowPatternExclusionMask & OverflowPatternExclusionKind::None)
-      return false;
-    if (OverflowPatternExclusionMask & OverflowPatternExclusionKind::All)
-      return true;
-    return OverflowPatternExclusionMask & Kind;
-  }
-
   /// Reset all of the options that are not considered when building a
   /// module.
   void resetNonModularOptions();
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index acc1f2fde53979a..c66e035a259b3ff 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2565,11 +2565,6 @@ defm sanitize_stats : BoolOption<"f", "sanitize-stats",
           "Disable">,
   BothFlags<[], [ClangOption], " sanitizer statistics gathering.">>,
   Group<f_clang_Group>;
-def fsanitize_overflow_pattern_exclusion_EQ : CommaJoined<["-"], "fsanitize-overflow-pattern-exclusion=">,
-  HelpText<"Specify the overflow patterns to exclude from artihmetic sanitizer instrumentation">,
-  Visibility<[ClangOption, CC1Option]>,
-  Values<"none,all,add-overflow-test,negated-unsigned-const,post-decr-while">,
-  MarshallingInfoStringVector<LangOpts<"OverflowPatternExclusionValues">>;
 def fsanitize_thread_memory_access : Flag<["-"], "fsanitize-thread-memory-access">,
                                      Group<f_clang_Group>,
                                      HelpText<"Enable memory access instrumentation in ThreadSanitizer (default)">;
@@ -5236,7 +5231,7 @@ let Flags = [TargetSpecific] in {
 def msse2avx : Flag<["-"], "msse2avx">, Group<m_Group>,
   Visibility<[ClangOption, CC1Option, CC1AsOption]>,
   HelpText<"Specify that the assembler should encode SSE instructions with VEX prefix">,
-  MarshallingInfoFlag<CodeGenOpts<"SSE2AVX">>;
+  MarshallingInfoFlag<CodeGenOpts<"X86Sse2Avx">>;
 } // let Flags = [TargetSpecific]
 
 defm zvector : BoolFOption<"zvector",
@@ -6149,6 +6144,10 @@ def mvis3 : Flag<["-"], "mvis3">, Group<m_sparc_Features_Group>;
 def mno_vis3 : Flag<["-"], "mno-vis3">, Group<m_sparc_Features_Group>;
 def mhard_quad_float : Flag<["-"], "mhard-quad-float">, Group<m_sparc_Features_Group>;
 def msoft_quad_float : Flag<["-"], "msoft-quad-float">, Group<m_sparc_Features_Group>;
+def mv8plus : Flag<["-"], "mv8plus">, Group<m_sparc_Features_Group>,
+  HelpText<"Enable V8+ mode, allowing use of 64-bit V9 instructions in 32-bit code">;
+def mno_v8plus : Flag<["-"], "mno-v8plus">, Group<m_sparc_Features_Group>,
+  HelpText<"Disable V8+ mode">;
 foreach i = 1 ... 7 in
   def ffixed_g#i : Flag<["-"], "ffixed-g"#i>, Group<m_sparc_Features_Group>,
     HelpText<"Reserve the G"#i#" register (SPARC only)">;
@@ -7134,7 +7133,7 @@ def crel : Flag<["--"], "crel">,
   MarshallingInfoFlag<CodeGenOpts<"Crel">>;
 def mrelax_relocations_no : Flag<["-"], "mrelax-relocations=no">,
     HelpText<"Disable x86 relax relocations">,
-    MarshallingInfoNegativeFlag<CodeGenOpts<"RelaxELFRelocations">>;
+    MarshallingInfoNegativeFlag<CodeGenOpts<"X86RelaxRelocations">>;
 def msave_temp_labels : Flag<["-"], "msave-temp-labels">,
   HelpText<"Save temporary labels in the symbol table. "
            "Note this may change .s semantics and shouldn't generally be used "
@@ -7960,7 +7959,7 @@ def fapply_global_visibility_to_externs : Flag<["-"], "fapply-global-visibility-
   MarshallingInfoFlag<LangOpts<"SetVisibilityForExternDecls">>;
 def fbracket_depth : Separate<["-"], "fbracket-depth">,
   HelpText<"Maximum nesting level for parentheses, brackets, and braces">,
-  MarshallingInfoInt<LangOpts<"BracketDepth">, "256">;
+  MarshallingInfoInt<LangOpts<"BracketDepth">, "2048">;
 defm const_strings : BoolOption<"f", "const-strings",
   LangOpts<"ConstStrings">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption, CC1Option], "Use">,
@@ -8841,7 +8840,7 @@ def dxil_validator_version : Option<["/", "-"], "validator-version", KIND_SEPARA
   Visibility<[DXCOption, ClangOption, CC1Option]>,
   HelpText<"Override validator version for module. Format: <major.minor>;"
            "Default: DXIL.dll version or current internal version">,
-  MarshallingInfoString<TargetOpts<"DxilValidatorVersion">>;
+  MarshallingInfoString<TargetOpts<"DxilValidatorVersion">, "\"1.8\"">;
 def target_profile : DXCJoinedOrSeparate<"T">, MetaVarName<"<profile>">,
   HelpText<"Set target profile">,
   Values<"ps_6_0, ps_6_1, ps_6_2, ps_6_3, ps_6_4, ps_6_5, ps_6_6, ps_6_7,"
diff --git a/clang/include/clang/Driver/SanitizerArgs.h b/clang/include/clang/Driver/SanitizerArgs.h
index e64ec463ca89070..47ef175302679f8 100644
--- a/clang/include/clang/Driver/SanitizerArgs.h
+++ b/clang/include/clang/Driver/SanitizerArgs.h
@@ -33,7 +33,6 @@ class SanitizerArgs {
   std::vector<std::string> BinaryMetadataIgnorelistFiles;
   int CoverageFeatures = 0;
   int BinaryMetadataFeatures = 0;
-  int OverflowPatternExclusions = 0;
   int MsanTrackOrigins = 0;
   bool MsanUseAfterDtor = true;
   bool MsanParamRetval = true;
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index ef6c76a070bfaaa..2af1d4065c3cc19 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -2858,7 +2858,8 @@ struct FormatStyle {
   PPDirectiveIndentStyle IndentPPDirectives;
 
   /// Indent the requires clause in a template. This only applies when
-  /// ``RequiresClausePosition`` is ``OwnLine``, or ``WithFollowing``.
+  /// ``RequiresClausePosition`` is ``OwnLine``, ``OwnLineWithBrace``,
+  /// or ``WithFollowing``.
   ///
   /// In clang-format 12, 13 and 14 it was named ``IndentRequires``.
   /// \code
@@ -3944,22 +3945,45 @@ struct FormatStyle {
   /// ``IndentRequires`` option is only used if the ``requires`` is put on the
   /// start of a line.
   enum RequiresClausePositionStyle : int8_t {
-    /// Always put the ``requires`` clause on its own line.
+    /// Always put the ``requires`` clause on its own line (possibly followed by
+    /// a semicolon).
     /// \code
     ///   template <typename T>
-    ///   requires C<T>
+    ///     requires C<T>
     ///   struct Foo {...
     ///
     ///   template <typename T>
-    ///   requires C<T>
+    ///   void bar(T t)
+    ///     requires C<T>;
+    ///
+    ///   template <typename T>
+    ///     requires C<T>
     ///   void bar(T t) {...
     ///
     ///   template <typename T>
     ///   void baz(T t)
-    ///   requires C<T>
+    ///     requires C<T>
     ///   {...
     /// \endcode
     RCPS_OwnLine,
+    /// As with ``OwnLine``, except, unless otherwise prohibited, place a
+    /// following open brace (of a function definition) to follow on the same
+    /// line.
+    /// \code
+    ///   void bar(T t)
+    ///     requires C<T> {
+    ///     return;
+    ///   }
+    ///
+    ///   void bar(T t)
+    ///     requires C<T> {}
+    ///
+    ///   template <typename T>
+    ///     requires C<T>
+    ///   void baz(T t) {
+    ///     ...
+    /// \endcode
+    RCPS_OwnLineWithBrace,
     /// Try to put the clause together with the preceding part of a declaration.
     /// For class templates: stick to the template declaration.
     /// For function templates: stick to the template declaration.
diff --git a/clang/include/clang/Rewrite/Core/DeltaTree.h b/clang/include/clang/Rewrite/Core/DeltaTree.h
deleted file mode 100644
index e566c92aaff91ad..000000000000000
--- a/clang/include/clang/Rewrite/Core/DeltaTree.h
+++ /dev/null
@@ -1,50 +0,0 @@
-//===- DeltaTree.h - B-Tree for Rewrite Delta tracking ----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the DeltaTree class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_REWRITE_CORE_DELTATREE_H
-#define LLVM_CLANG_REWRITE_CORE_DELTATREE_H
-
-namespace clang {
-
-  /// DeltaTree - a multiway search tree (BTree) structure with some fancy
-  /// features.  B-Trees are generally more memory and cache efficient than
-  /// binary trees, because they store multiple keys/values in each node.  This
-  /// implements a key/value mapping from index to delta, and allows fast lookup
-  /// on index.  However, an added (important) bonus is that it can also
-  /// efficiently tell us the full accumulated delta for a specific file offset
-  /// as well, without traversing the whole tree.
-  class DeltaTree {
-    void *Root;    // "DeltaTreeNode *"
-
-  public:
-    DeltaTree();
-
-    // Note: Currently we only support copying when the RHS is empty.
-    DeltaTree(const DeltaTree &RHS);
-
-    DeltaTree &operator=(const DeltaTree &) = delete;
-    ~DeltaTree();
-
-    /// getDeltaAt - Return the accumulated delta at the specified file offset.
-    /// This includes all insertions or delections that occurred *before* the
-    /// specified file index.
-    int getDeltaAt(unsigned FileIndex) const;
-
-    /// AddDelta - When a change is made that shifts around the text buffer,
-    /// this method is used to record that info.  It inserts a delta of 'Delta'
-    /// into the current DeltaTree at offset FileIndex.
-    void AddDelta(unsigned FileIndex, int Delta);
-  };
-
-} // namespace clang
-
-#endif // LLVM_CLANG_REWRITE_CORE_DELTATREE_H
diff --git a/clang/include/clang/Rewrite/Core/HTMLRewrite.h b/clang/include/clang/Rewrite/Core/HTMLRewrite.h
index eecf589632746c1..9edb514d566b9c1 100644
--- a/clang/include/clang/Rewrite/Core/HTMLRewrite.h
+++ b/clang/include/clang/Rewrite/Core/HTMLRewrite.h
@@ -17,10 +17,13 @@
 #include "clang/Basic/SourceLocation.h"
 #include <string>
 
+namespace llvm {
+class RewriteBuffer;
+} // namespace llvm
+
 namespace clang {
 
 class Rewriter;
-class RewriteBuffer;
 class Preprocessor;
 
 namespace html {
@@ -53,9 +56,9 @@ namespace html {
 
   /// HighlightRange - This is the same as the above method, but takes
   /// decomposed file locations.
-  void HighlightRange(RewriteBuffer &RB, unsigned B, unsigned E,
-                      const char *BufferStart,
-                      const char *StartTag, const char *EndTag);
+  void HighlightRange(llvm::RewriteBuffer &RB, unsigned B, unsigned E,
+                      const char *BufferStart, const char *StartTag,
+                      const char *EndTag);
 
   /// EscapeText - HTMLize a specified file so that special characters are
   /// are translated so that they are not interpreted as HTML tags.
diff --git a/clang/include/clang/Rewrite/Core/RewriteRope.h b/clang/include/clang/Rewrite/Core/RewriteRope.h
deleted file mode 100644
index 73e66e111f57434..000000000000000
--- a/clang/include/clang/Rewrite/Core/RewriteRope.h
+++ /dev/null
@@ -1,223 +0,0 @@
-//===- RewriteRope.h - Rope specialized for rewriter ------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the RewriteRope class, which is a powerful string class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_REWRITE_CORE_REWRITEROPE_H
-#define LLVM_CLANG_REWRITE_CORE_REWRITEROPE_H
-
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include "llvm/ADT/StringRef.h"
-#include <cassert>
-#include <cstddef>
-#include <iterator>
-#include <utility>
-
-namespace clang {
-
-  //===--------------------------------------------------------------------===//
-  // RopeRefCountString Class
-  //===--------------------------------------------------------------------===//
-
-  /// RopeRefCountString - This struct is allocated with 'new char[]' from the
-  /// heap, and represents a reference counted chunk of string data.  When its
-  /// ref count drops to zero, it is delete[]'d.  This is primarily managed
-  /// through the RopePiece class below.
-  struct RopeRefCountString {
-    unsigned RefCount;
-    char Data[1];  //  Variable sized.
-
-    void Retain() { ++RefCount; }
-
-    void Release() {
-      assert(RefCount > 0 && "Reference count is already zero.");
-      if (--RefCount == 0)
-        delete [] (char*)this;
-    }
-  };
-
-  //===--------------------------------------------------------------------===//
-  // RopePiece Class
-  //===--------------------------------------------------------------------===//
-
-  /// RopePiece - This class represents a view into a RopeRefCountString object.
-  /// This allows references to string data to be efficiently chopped up and
-  /// moved around without having to push around the string data itself.
-  ///
-  /// For example, we could have a 1M RopePiece and want to insert something
-  /// into the middle of it.  To do this, we split it into two RopePiece objects
-  /// that both refer to the same underlying RopeRefCountString (just with
-  /// different offsets) which is a nice constant time operation.
-  struct RopePiece {
-    llvm::IntrusiveRefCntPtr<RopeRefCountString> StrData;
-    unsigned StartOffs = 0;
-    unsigned EndOffs = 0;
-
-    RopePiece() = default;
-    RopePiece(llvm::IntrusiveRefCntPtr<RopeRefCountString> Str, unsigned Start,
-              unsigned End)
-        : StrData(std::move(Str)), StartOffs(Start), EndOffs(End) {}
-
-    const char &operator[](unsigned Offset) const {
-      return StrData->Data[Offset+StartOffs];
-    }
-    char &operator[](unsigned Offset) {
-      return StrData->Data[Offset+StartOffs];
-    }
-
-    unsigned size() const { return EndOffs-StartOffs; }
-  };
-
-  //===--------------------------------------------------------------------===//
-  // RopePieceBTreeIterator Class
-  //===--------------------------------------------------------------------===//
-
-  /// RopePieceBTreeIterator - This class provides read-only forward iteration
-  /// over bytes that are in a RopePieceBTree.  This first iterates over bytes
-  /// in a RopePiece, then iterates over RopePiece's in a RopePieceBTreeLeaf,
-  /// then iterates over RopePieceBTreeLeaf's in a RopePieceBTree.
-  class RopePieceBTreeIterator {
-    /// CurNode - The current B+Tree node that we are inspecting.
-    const void /*RopePieceBTreeLeaf*/ *CurNode = nullptr;
-
-    /// CurPiece - The current RopePiece in the B+Tree node that we're
-    /// inspecting.
-    const RopePiece *CurPiece = nullptr;
-
-    /// CurChar - The current byte in the RopePiece we are pointing to.
-    unsigned CurChar = 0;
-
-  public:
-    using iterator_category = std::forward_iterator_tag;
-    using value_type = const char;
-    using difference_type = std::ptrdiff_t;
-    using pointer = value_type *;
-    using reference = value_type &;
-
-    RopePieceBTreeIterator() = default;
-    RopePieceBTreeIterator(const void /*RopePieceBTreeNode*/ *N);
-
-    char operator*() const {
-      return (*CurPiece)[CurChar];
-    }
-
-    bool operator==(const RopePieceBTreeIterator &RHS) const {
-      return CurPiece == RHS.CurPiece && CurChar == RHS.CurChar;
-    }
-    bool operator!=(const RopePieceBTreeIterator &RHS) const {
-      return !operator==(RHS);
-    }
-
-    RopePieceBTreeIterator& operator++() {   // Preincrement
-      if (CurChar+1 < CurPiece->size())
-        ++CurChar;
-      else
-        MoveToNextPiece();
-      return *this;
-    }
-
-    RopePieceBTreeIterator operator++(int) { // Postincrement
-      RopePieceBTreeIterator tmp = *this; ++*this; return tmp;
-    }
-
-    llvm::StringRef piece() const {
-      return llvm::StringRef(&(*CurPiece)[0], CurPiece->size());
-    }
-
-    void MoveToNextPiece();
-  };
-
-  //===--------------------------------------------------------------------===//
-  // RopePieceBTree Class
-  //===--------------------------------------------------------------------===//
-
-  class RopePieceBTree {
-    void /*RopePieceBTreeNode*/ *Root;
-
-  public:
-    RopePieceBTree();
-    RopePieceBTree(const RopePieceBTree &RHS);
-    RopePieceBTree &operator=(const RopePieceBTree &) = delete;
-    ~RopePieceBTree();
-
-    using iterator = RopePieceBTreeIterator;
-
-    iterator begin() const { return iterator(Root); }
-    iterator end() const { return iterator(); }
-    unsigned size() const;
-    unsigned empty() const { return size() == 0; }
-
-    void clear();
-
-    void insert(unsigned Offset, const RopePiece &R);
-
-    void erase(unsigned Offset, unsigned NumBytes);
-  };
-
-  //===--------------------------------------------------------------------===//
-  // RewriteRope Class
-  //===--------------------------------------------------------------------===//
-
-/// RewriteRope - A powerful string class.  This class supports extremely
-/// efficient insertions and deletions into the middle of it, even for
-/// ridiculously long strings.
-class RewriteRope {
-  RopePieceBTree Chunks;
-
-  /// We allocate space for string data out of a buffer of size AllocChunkSize.
-  /// This keeps track of how much space is left.
-  llvm::IntrusiveRefCntPtr<RopeRefCountString> AllocBuffer;
-  enum { AllocChunkSize = 4080 };
-  unsigned AllocOffs = AllocChunkSize;
-
-public:
-  RewriteRope() = default;
-  RewriteRope(const RewriteRope &RHS) : Chunks(RHS.Chunks) {}
-
-  // The copy assignment operator is defined as deleted pending further
-  // motivation.
-  RewriteRope &operator=(const RewriteRope &) = delete;
-
-  using iterator = RopePieceBTree::iterator;
-  using const_iterator = RopePieceBTree::iterator;
-
-  iterator begin() const { return Chunks.begin(); }
-  iterator end() const { return Chunks.end(); }
-  unsigned size() const { return Chunks.size(); }
-
-  void clear() {
-    Chunks.clear();
-  }
-
-  void assign(const char *Start, const char *End) {
-    clear();
-    if (Start != End)
-      Chunks.insert(0, MakeRopeString(Start, End));
-  }
-
-  void insert(unsigned Offset, const char *Start, const char *End) {
-    assert(Offset <= size() && "Invalid position to insert!");
-    if (Start == End) return;
-    Chunks.insert(Offset, MakeRopeString(Start, End));
-  }
-
-  void erase(unsigned Offset, unsigned NumBytes) {
-    assert(Offset+NumBytes <= size() && "Invalid region to erase!");
-    if (NumBytes == 0) return;
-    Chunks.erase(Offset, NumBytes);
-  }
-
-private:
-  RopePiece MakeRopeString(const char *Start, const char *End);
-};
-
-} // namespace clang
-
-#endif // LLVM_CLANG_REWRITE_CORE_REWRITEROPE_H
diff --git a/clang/include/clang/Rewrite/Core/Rewriter.h b/clang/include/clang/Rewrite/Core/Rewriter.h
index c89015e4055820c..4e96f6fcca919ce 100644
--- a/clang/include/clang/Rewrite/Core/Rewriter.h
+++ b/clang/include/clang/Rewrite/Core/Rewriter.h
@@ -16,7 +16,7 @@
 
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Rewrite/Core/RewriteBuffer.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/StringRef.h"
 #include <map>
 #include <string>
@@ -32,7 +32,7 @@ class SourceManager;
 class Rewriter {
   SourceManager *SourceMgr = nullptr;
   const LangOptions *LangOpts = nullptr;
-  std::map<FileID, RewriteBuffer> RewriteBuffers;
+  std::map<FileID, llvm::RewriteBuffer> RewriteBuffers;
 
 public:
   struct RewriteOptions {
@@ -49,7 +49,7 @@ class Rewriter {
     ///
     /// FIXME: This sometimes corrupts the file's rewrite buffer due to
     /// incorrect indexing in the implementation (see the FIXME in
-    /// clang::RewriteBuffer::RemoveText).  Moreover, it's inefficient because
+    /// llvm::RewriteBuffer::RemoveText).  Moreover, it's inefficient because
     /// it must scan the buffer from the beginning to find the start of the
     /// line.  When feasible, it's better for the caller to check for a blank
     /// line and then, if found, expand the removal range to include it.
@@ -62,8 +62,9 @@ class Rewriter {
     RewriteOptions() {}
   };
 
-  using buffer_iterator = std::map<FileID, RewriteBuffer>::iterator;
-  using const_buffer_iterator = std::map<FileID, RewriteBuffer>::const_iterator;
+  using buffer_iterator = std::map<FileID, llvm::RewriteBuffer>::iterator;
+  using const_buffer_iterator =
+      std::map<FileID, llvm::RewriteBuffer>::const_iterator;
 
   explicit Rewriter() = default;
   explicit Rewriter(SourceManager &SM, const LangOptions &LO)
@@ -191,13 +192,13 @@ class Rewriter {
   /// buffer, and allows you to write on it directly.  This is useful if you
   /// want efficient low-level access to apis for scribbling on one specific
   /// FileID's buffer.
-  RewriteBuffer &getEditBuffer(FileID FID);
+  llvm::RewriteBuffer &getEditBuffer(FileID FID);
 
   /// getRewriteBufferFor - Return the rewrite buffer for the specified FileID.
   /// If no modification has been made to it, return null.
-  const RewriteBuffer *getRewriteBufferFor(FileID FID) const {
-    std::map<FileID, RewriteBuffer>::const_iterator I =
-      RewriteBuffers.find(FID);
+  const llvm::RewriteBuffer *getRewriteBufferFor(FileID FID) const {
+    std::map<FileID, llvm::RewriteBuffer>::const_iterator I =
+        RewriteBuffers.find(FID);
     return I == RewriteBuffers.end() ? nullptr : &I->second;
   }
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index a025ff6fc13f360..299a916b9abf8d9 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15,36 +15,42 @@
 #define LLVM_CLANG_SEMA_SEMA_H
 
 #include "clang/APINotes/APINotesManager.h"
-#include "clang/AST/ASTConcept.h"
 #include "clang/AST/ASTFwd.h"
 #include "clang/AST/Attr.h"
-#include "clang/AST/Availability.h"
-#include "clang/AST/ComparisonCategories.h"
+#include "clang/AST/AttrIterator.h"
+#include "clang/AST/CharUnits.h"
+#include "clang/AST/DeclBase.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprConcepts.h"
-#include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExternalASTSource.h"
-#include "clang/AST/LocInfoType.h"
-#include "clang/AST/MangleNumberingContext.h"
-#include "clang/AST/NSAPI.h"
-#include "clang/AST/PrettyPrinter.h"
+#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/OperationKinds.h"
 #include "clang/AST/StmtCXX.h"
+#include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
-#include "clang/AST/TypeOrdering.h"
-#include "clang/Basic/BitmaskEnum.h"
+#include "clang/Basic/AttrSubjectMatchRules.h"
 #include "clang/Basic/Builtins.h"
+#include "clang/Basic/CapturedStmt.h"
 #include "clang/Basic/Cuda.h"
-#include "clang/Basic/DarwinSDKInfo.h"
+#include "clang/Basic/DiagnosticSema.h"
+#include "clang/Basic/ExceptionSpecificationType.h"
 #include "clang/Basic/ExpressionTraits.h"
-#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/Lambda.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/OpenCLOptions.h"
+#include "clang/Basic/OperatorKinds.h"
+#include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/PragmaKinds.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TemplateKinds.h"
+#include "clang/Basic/TokenKinds.h"
 #include "clang/Basic/TypeTraits.h"
 #include "clang/Sema/AnalysisBasedWarnings.h"
 #include "clang/Sema/Attr.h"
@@ -52,124 +58,100 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/ExternalSemaSource.h"
 #include "clang/Sema/IdentifierResolver.h"
-#include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/Ownership.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Redeclaration.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
-#include "clang/Sema/SemaConcept.h"
-#include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "clang/Sema/Weak.h"
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/PointerUnion.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cassert>
+#include <climits>
+#include <cstddef>
+#include <cstdint>
 #include <deque>
+#include <functional>
+#include <iterator>
 #include <memory>
 #include <optional>
 #include <string>
 #include <tuple>
+#include <type_traits>
+#include <utility>
 #include <vector>
 
 namespace llvm {
-class APSInt;
-template <typename ValueT, typename ValueInfoT> class DenseSet;
-class SmallBitVector;
 struct InlineAsmIdentifierInfo;
 } // namespace llvm
 
 namespace clang {
 class ADLResult;
+class APValue;
+struct ASTConstraintSatisfaction;
 class ASTConsumer;
 class ASTContext;
+class ASTDeclReader;
 class ASTMutationListener;
 class ASTReader;
 class ASTWriter;
-class ArrayType;
-class ParsedAttr;
-class BindingDecl;
-class BlockDecl;
-class CapturedDecl;
 class CXXBasePath;
 class CXXBasePaths;
-class CXXBindTemporaryExpr;
-typedef SmallVector<CXXBaseSpecifier *, 4> CXXCastPath;
-class CXXConstructorDecl;
-class CXXConversionDecl;
-class CXXDeleteExpr;
-class CXXDestructorDecl;
 class CXXFieldCollector;
-class CXXMemberCallExpr;
-class CXXMethodDecl;
-class CXXScopeSpec;
-class CXXTemporary;
-class CXXTryStmt;
-class CallExpr;
-class ClassTemplateDecl;
-class ClassTemplatePartialSpecializationDecl;
-class ClassTemplateSpecializationDecl;
-class VarTemplatePartialSpecializationDecl;
 class CodeCompleteConsumer;
-class CodeCompletionAllocator;
-class CodeCompletionTUInfo;
-class CodeCompletionResult;
-class CoroutineBodyStmt;
-class Decl;
-class DeclAccessPair;
-class DeclContext;
-class DeclRefExpr;
-class DeclaratorDecl;
+enum class ComparisonCategoryType : unsigned char;
+class ConstraintSatisfaction;
+class DarwinSDKInfo;
+class DeclGroupRef;
 class DeducedTemplateArgument;
+struct DeductionFailureInfo;
 class DependentDiagnostic;
-class DesignatedInitExpr;
 class Designation;
-class EnableIfAttr;
-class EnumConstantDecl;
-class Expr;
-class ExtVectorType;
-class FormatAttr;
-class FriendDecl;
-class FunctionDecl;
-class FunctionProtoType;
-class FunctionTemplateDecl;
+class IdentifierInfo;
 class ImplicitConversionSequence;
 typedef MutableArrayRef<ImplicitConversionSequence> ConversionSequenceList;
-class InitListExpr;
 class InitializationKind;
 class InitializationSequence;
 class InitializedEntity;
-class IntegerLiteral;
-class LabelStmt;
-class LambdaExpr;
-class LangOptions;
+enum class LangAS : unsigned int;
 class LocalInstantiationScope;
 class LookupResult;
-class MacroInfo;
+class MangleNumberingContext;
 typedef ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> ModuleIdPath;
 class ModuleLoader;
 class MultiLevelTemplateArgumentList;
-class NamedDecl;
-class ObjCImplementationDecl;
+struct NormalizedConstraint;
 class ObjCInterfaceDecl;
 class ObjCMethodDecl;
-class ObjCProtocolDecl;
 struct OverloadCandidate;
 enum class OverloadCandidateParamOrder : char;
 enum OverloadCandidateRewriteKind : unsigned;
 class OverloadCandidateSet;
-class OverloadExpr;
-class ParenListExpr;
-class ParmVarDecl;
 class Preprocessor;
-class PseudoDestructorTypeStorage;
-class PseudoObjectExpr;
-class QualType;
 class SemaAMDGPU;
 class SemaARM;
 class SemaAVR;
@@ -196,41 +178,19 @@ class SemaSystemZ;
 class SemaWasm;
 class SemaX86;
 class StandardConversionSequence;
-class Stmt;
-class StringLiteral;
-class SwitchStmt;
 class TemplateArgument;
-class TemplateArgumentList;
 class TemplateArgumentLoc;
-class TemplateDecl;
 class TemplateInstantiationCallback;
-class TemplateParameterList;
 class TemplatePartialOrderingContext;
-class TemplateTemplateParmDecl;
+class TemplateSpecCandidateSet;
 class Token;
-class TypeAliasDecl;
-class TypedefDecl;
-class TypedefNameDecl;
-class TypeLoc;
+class TypeConstraint;
 class TypoCorrectionConsumer;
-class UnqualifiedId;
-class UnresolvedLookupExpr;
-class UnresolvedMemberExpr;
 class UnresolvedSetImpl;
 class UnresolvedSetIterator;
-class UsingDecl;
-class UsingShadowDecl;
-class ValueDecl;
-class VarDecl;
-class VarTemplateSpecializationDecl;
-class VisibilityAttr;
 class VisibleDeclConsumer;
-class IndirectFieldDecl;
-struct DeductionFailureInfo;
-class TemplateSpecCandidateSet;
 
 namespace sema {
-class AccessedEntity;
 class BlockScopeInfo;
 class Capture;
 class CapturedRegionScopeInfo;
@@ -240,8 +200,6 @@ class DelayedDiagnostic;
 class DelayedDiagnosticPool;
 class FunctionScopeInfo;
 class LambdaScopeInfo;
-class PossiblyUnreachableDiag;
-class RISCVIntrinsicManager;
 class SemaPPCallbacks;
 class TemplateDeductionInfo;
 } // namespace sema
@@ -536,38 +494,40 @@ class Sema final : public SemaBase {
   // Table of Contents
   // -----------------
   // 1. Semantic Analysis (Sema.cpp)
-  // 2. C++ Access Control (SemaAccess.cpp)
-  // 3. Attributes (SemaAttr.cpp)
-  // 4. Availability Attribute Handling (SemaAvailability.cpp)
-  // 5. Casts (SemaCast.cpp)
-  // 6. Extra Semantic Checking (SemaChecking.cpp)
-  // 7. C++ Coroutines (SemaCoroutine.cpp)
-  // 8. C++ Scope Specifiers (SemaCXXScopeSpec.cpp)
-  // 9. Declarations (SemaDecl.cpp)
-  // 10. Declaration Attribute Handling (SemaDeclAttr.cpp)
-  // 11. C++ Declarations (SemaDeclCXX.cpp)
-  // 12. C++ Exception Specifications (SemaExceptionSpec.cpp)
-  // 13. Expressions (SemaExpr.cpp)
-  // 14. C++ Expressions (SemaExprCXX.cpp)
-  // 15. Member Access Expressions (SemaExprMember.cpp)
-  // 16. Initializers (SemaInit.cpp)
-  // 17. C++ Lambda Expressions (SemaLambda.cpp)
-  // 18. Name Lookup (SemaLookup.cpp)
-  // 19. Modules (SemaModule.cpp)
-  // 20. C++ Overloading (SemaOverload.cpp)
-  // 21. Statements (SemaStmt.cpp)
-  // 22. `inline asm` Statement (SemaStmtAsm.cpp)
-  // 23. Statement Attribute Handling (SemaStmtAttr.cpp)
-  // 24. C++ Templates (SemaTemplate.cpp)
-  // 25. C++ Template Argument Deduction (SemaTemplateDeduction.cpp)
-  // 26. C++ Template Deduction Guide (SemaTemplateDeductionGuide.cpp)
-  // 27. C++ Template Instantiation (SemaTemplateInstantiate.cpp)
-  // 28. C++ Template Declaration Instantiation
+  // 2. API Notes (SemaAPINotes.cpp)
+  // 3. C++ Access Control (SemaAccess.cpp)
+  // 4. Attributes (SemaAttr.cpp)
+  // 5. Availability Attribute Handling (SemaAvailability.cpp)
+  // 6. Bounds Safety (SemaBoundsSafety.cpp)
+  // 7. Casts (SemaCast.cpp)
+  // 8. Extra Semantic Checking (SemaChecking.cpp)
+  // 9. C++ Coroutines (SemaCoroutine.cpp)
+  // 10. C++ Scope Specifiers (SemaCXXScopeSpec.cpp)
+  // 11. Declarations (SemaDecl.cpp)
+  // 12. Declaration Attribute Handling (SemaDeclAttr.cpp)
+  // 13. C++ Declarations (SemaDeclCXX.cpp)
+  // 14. C++ Exception Specifications (SemaExceptionSpec.cpp)
+  // 15. Expressions (SemaExpr.cpp)
+  // 16. C++ Expressions (SemaExprCXX.cpp)
+  // 17. Member Access Expressions (SemaExprMember.cpp)
+  // 18. Initializers (SemaInit.cpp)
+  // 19. C++ Lambda Expressions (SemaLambda.cpp)
+  // 20. Name Lookup (SemaLookup.cpp)
+  // 21. Modules (SemaModule.cpp)
+  // 22. C++ Overloading (SemaOverload.cpp)
+  // 23. Statements (SemaStmt.cpp)
+  // 24. `inline asm` Statement (SemaStmtAsm.cpp)
+  // 25. Statement Attribute Handling (SemaStmtAttr.cpp)
+  // 26. C++ Templates (SemaTemplate.cpp)
+  // 27. C++ Template Argument Deduction (SemaTemplateDeduction.cpp)
+  // 28. C++ Template Deduction Guide (SemaTemplateDeductionGuide.cpp)
+  // 29. C++ Template Instantiation (SemaTemplateInstantiate.cpp)
+  // 30. C++ Template Declaration Instantiation
   //     (SemaTemplateInstantiateDecl.cpp)
-  // 29. C++ Variadic Templates (SemaTemplateVariadic.cpp)
-  // 30. Constraints and Concepts (SemaConcept.cpp)
-  // 31. Types (SemaType.cpp)
-  // 32. FixIt Helpers (SemaFixItUtils.cpp)
+  // 31. C++ Variadic Templates (SemaTemplateVariadic.cpp)
+  // 32. Constraints and Concepts (SemaConcept.cpp)
+  // 33. Types (SemaType.cpp)
+  // 34. FixIt Helpers (SemaFixItUtils.cpp)
 
   /// \name Semantic Analysis
   /// Implementations are in Sema.cpp
@@ -1325,6 +1285,25 @@ class Sema final : public SemaBase {
   //
   //
 
+  /// \name API Notes
+  /// Implementations are in SemaAPINotes.cpp
+  ///@{
+
+public:
+  /// Map any API notes provided for this declaration to attributes on the
+  /// declaration.
+  ///
+  /// Triggered by declaration-attribute processing.
+  void ProcessAPINotes(Decl *D);
+
+  ///@}
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
   /// \name C++ Access Control
   /// Implementations are in SemaAccess.cpp
   ///@{
@@ -2095,6 +2074,39 @@ class Sema final : public SemaBase {
   //
   //
 
+  /// \name Bounds Safety
+  /// Implementations are in SemaBoundsSafety.cpp
+  ///@{
+public:
+  /// Check if applying the specified attribute variant from the "counted by"
+  /// family of attributes to FieldDecl \p FD is semantically valid. If
+  /// semantically invalid diagnostics will be emitted explaining the problems.
+  ///
+  /// \param FD The FieldDecl to apply the attribute to
+  /// \param E The count expression on the attribute
+  /// \param CountInBytes If true the attribute is from the "sized_by" family of
+  ///                     attributes. If the false the attribute is from
+  ///                     "counted_by" family of attributes.
+  /// \param OrNull If true the attribute is from the "_or_null" suffixed family
+  ///               of attributes. If false the attribute does not have the
+  ///               suffix.
+  ///
+  /// Together \p CountInBytes and \p OrNull decide the attribute variant. E.g.
+  /// \p CountInBytes and \p OrNull both being true indicates the
+  /// `counted_by_or_null` attribute.
+  ///
+  /// \returns false iff semantically valid.
+  bool CheckCountedByAttrOnField(FieldDecl *FD, Expr *E, bool CountInBytes,
+                                 bool OrNull);
+
+  ///@}
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
   /// \name Casts
   /// Implementations are in SemaCast.cpp
   ///@{
@@ -3800,7 +3812,8 @@ class Sema final : public SemaBase {
                                    const ParsedAttributesView &DeclAttrs,
                                    MultiTemplateParamsArg TemplateParams,
                                    bool IsExplicitInstantiation,
-                                   RecordDecl *&AnonRecord);
+                                   RecordDecl *&AnonRecord,
+                                   SourceLocation EllipsisLoc = {});
 
   /// BuildAnonymousStructOrUnion - Handle the declaration of an
   /// anonymous structure or union. Anonymous unions are a C++ feature
@@ -5538,7 +5551,8 @@ class Sema final : public SemaBase {
   /// parameters present at all, require proper matching, i.e.
   ///   template <> template \<class T> friend class A<int>::B;
   Decl *ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
-                            MultiTemplateParamsArg TemplateParams);
+                            MultiTemplateParamsArg TemplateParams,
+                            SourceLocation EllipsisLoc);
   NamedDecl *ActOnFriendFunctionDecl(Scope *S, Declarator &D,
                                      MultiTemplateParamsArg TemplateParams);
 
@@ -5852,6 +5866,7 @@ class Sema final : public SemaBase {
                                      unsigned TagSpec, SourceLocation TagLoc,
                                      CXXScopeSpec &SS, IdentifierInfo *Name,
                                      SourceLocation NameLoc,
+                                     SourceLocation EllipsisLoc,
                                      const ParsedAttributesView &Attr,
                                      MultiTemplateParamsArg TempParamLists);
 
@@ -15042,58 +15057,6 @@ class Sema final : public SemaBase {
   std::string getFixItZeroLiteralForType(QualType T, SourceLocation Loc) const;
 
   ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
-  /// \name API Notes
-  /// Implementations are in SemaAPINotes.cpp
-  ///@{
-
-public:
-  /// Map any API notes provided for this declaration to attributes on the
-  /// declaration.
-  ///
-  /// Triggered by declaration-attribute processing.
-  void ProcessAPINotes(Decl *D);
-
-  ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
-  /// \name Bounds Safety
-  /// Implementations are in SemaBoundsSafety.cpp
-  ///@{
-public:
-  /// Check if applying the specified attribute variant from the "counted by"
-  /// family of attributes to FieldDecl \p FD is semantically valid. If
-  /// semantically invalid diagnostics will be emitted explaining the problems.
-  ///
-  /// \param FD The FieldDecl to apply the attribute to
-  /// \param E The count expression on the attribute
-  /// \param CountInBytes If true the attribute is from the "sized_by" family of
-  ///                     attributes. If the false the attribute is from
-  ///                     "counted_by" family of attributes.
-  /// \param OrNull If true the attribute is from the "_or_null" suffixed family
-  ///               of attributes. If false the attribute does not have the
-  ///               suffix.
-  ///
-  /// Together \p CountInBytes and \p OrNull decide the attribute variant. E.g.
-  /// \p CountInBytes and \p OrNull both being true indicates the
-  /// `counted_by_or_null` attribute.
-  ///
-  /// \returns false iff semantically valid.
-  bool CheckCountedByAttrOnField(FieldDecl *FD, Expr *E, bool CountInBytes,
-                                 bool OrNull);
-
-  ///@}
 };
 
 DeductionFailureInfo
diff --git a/clang/include/clang/Sema/SemaAMDGPU.h b/clang/include/clang/Sema/SemaAMDGPU.h
index 969078f552c6ab7..3fdb39202610d04 100644
--- a/clang/include/clang/Sema/SemaAMDGPU.h
+++ b/clang/include/clang/Sema/SemaAMDGPU.h
@@ -13,14 +13,13 @@
 #ifndef LLVM_CLANG_SEMA_SEMAAMDGPU_H
 #define LLVM_CLANG_SEMA_SEMAAMDGPU_H
 
-#include "clang/AST/Attr.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/AttributeCommonInfo.h"
-#include "clang/Sema/ParsedAttr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class AttributeCommonInfo;
+class ParsedAttr;
+
 class SemaAMDGPU : public SemaBase {
 public:
   SemaAMDGPU(Sema &S);
diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h
index fedc7df7908f17c..b8196a3170d63c6 100644
--- a/clang/include/clang/Sema/SemaARM.h
+++ b/clang/include/clang/Sema/SemaARM.h
@@ -13,15 +13,18 @@
 #ifndef LLVM_CLANG_SEMA_SEMAARM_H
 #define LLVM_CLANG_SEMA_SEMAARM_H
 
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
 #include <tuple>
 
+namespace llvm {
+template <typename T, unsigned N> class SmallVector;
+} // namespace llvm
+
 namespace clang {
 class ParsedAttr;
+class TargetInfo;
 
 class SemaARM : public SemaBase {
 public:
@@ -60,10 +63,10 @@ class SemaARM : public SemaBase {
                             unsigned ExpectedFieldNum, bool AllowName);
   bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
 
-  bool MveAliasValid(unsigned BuiltinID, StringRef AliasName);
-  bool CdeAliasValid(unsigned BuiltinID, StringRef AliasName);
-  bool SveAliasValid(unsigned BuiltinID, StringRef AliasName);
-  bool SmeAliasValid(unsigned BuiltinID, StringRef AliasName);
+  bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
+  bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
+  bool SveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
+  bool SmeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
   void handleBuiltinAliasAttr(Decl *D, const ParsedAttr &AL);
   void handleNewAttr(Decl *D, const ParsedAttr &AL);
   void handleCmseNSEntryAttr(Decl *D, const ParsedAttr &AL);
diff --git a/clang/include/clang/Sema/SemaAVR.h b/clang/include/clang/Sema/SemaAVR.h
index 708da3a6026acee..d89ce054b864d3e 100644
--- a/clang/include/clang/Sema/SemaAVR.h
+++ b/clang/include/clang/Sema/SemaAVR.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CLANG_SEMA_SEMAAVR_H
 #define LLVM_CLANG_SEMA_SEMAAVR_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
-class Decl;
 class ParsedAttr;
 
 class SemaAVR : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaBPF.h b/clang/include/clang/Sema/SemaBPF.h
index 0182ccfe508a7be..d7577aea7012274 100644
--- a/clang/include/clang/Sema/SemaBPF.h
+++ b/clang/include/clang/Sema/SemaBPF.h
@@ -13,9 +13,7 @@
 #ifndef LLVM_CLANG_SEMA_SEMABPF_H
 #define LLVM_CLANG_SEMA_SEMABPF_H
 
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
diff --git a/clang/include/clang/Sema/SemaCUDA.h b/clang/include/clang/Sema/SemaCUDA.h
index 63dc3f4da240b36..71f05e88fb539cf 100644
--- a/clang/include/clang/Sema/SemaCUDA.h
+++ b/clang/include/clang/Sema/SemaCUDA.h
@@ -13,24 +13,34 @@
 #ifndef LLVM_CLANG_SEMA_SEMACUDA_H
 #define LLVM_CLANG_SEMA_SEMACUDA_H
 
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclCXX.h"
+#include "clang/AST/ASTFwd.h"
+#include "clang/AST/DeclAccessPair.h"
 #include "clang/AST/Redeclarable.h"
 #include "clang/Basic/Cuda.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Ownership.h"
-#include "clang/Sema/ParsedAttr.h"
-#include "clang/Sema/Scope.h"
-#include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/SmallVector.h"
 #include <string>
+#include <utility>
 
 namespace clang {
+namespace sema {
+class Capture;
+} // namespace sema
 
+class ASTReader;
+class ASTWriter;
 enum class CUDAFunctionTarget;
+enum class CXXSpecialMemberKind;
+class ParsedAttributesView;
+class Scope;
 
 class SemaCUDA : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaCodeCompletion.h b/clang/include/clang/Sema/SemaCodeCompletion.h
index 6bf0f2473cbf437..50409439389b068 100644
--- a/clang/include/clang/Sema/SemaCodeCompletion.h
+++ b/clang/include/clang/Sema/SemaCodeCompletion.h
@@ -13,26 +13,26 @@
 #ifndef LLVM_CLANG_SEMA_SEMACODECOMPLETION_H
 #define LLVM_CLANG_SEMA_SEMACODECOMPLETION_H
 
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclCXX.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/AttributeCommonInfo.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/ModuleLoader.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Designator.h"
 #include "clang/Sema/Ownership.h"
-#include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/StringRef.h"
 #include <optional>
 
 namespace clang {
+class DeclGroupRef;
+class MacroInfo;
+class Scope;
+class TemplateName;
 
 class SemaCodeCompletion : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index d60cb2a57d49187..3aae3383c215b55 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -13,19 +13,18 @@
 #ifndef LLVM_CLANG_SEMA_SEMAHLSL_H
 #define LLVM_CLANG_SEMA_SEMAHLSL_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Attr.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/AttributeCommonInfo.h"
-#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
+#include "llvm/TargetParser/Triple.h"
 #include <initializer_list>
 
 namespace clang {
+class AttributeCommonInfo;
+class IdentifierInfo;
 class ParsedAttr;
+class Scope;
 
 class SemaHLSL : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaHexagon.h b/clang/include/clang/Sema/SemaHexagon.h
index 2d4a04f824bc2ef..82e811dfc42e484 100644
--- a/clang/include/clang/Sema/SemaHexagon.h
+++ b/clang/include/clang/Sema/SemaHexagon.h
@@ -13,7 +13,7 @@
 #ifndef LLVM_CLANG_SEMA_SEMAHEXAGON_H
 #define LLVM_CLANG_SEMA_SEMAHEXAGON_H
 
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
diff --git a/clang/include/clang/Sema/SemaLoongArch.h b/clang/include/clang/Sema/SemaLoongArch.h
index aef0df9e8710ff0..98a7e99e31572a2 100644
--- a/clang/include/clang/Sema/SemaLoongArch.h
+++ b/clang/include/clang/Sema/SemaLoongArch.h
@@ -13,11 +13,12 @@
 #ifndef LLVM_CLANG_SEMA_SEMALOONGARCH_H
 #define LLVM_CLANG_SEMA_SEMALOONGARCH_H
 
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class TargetInfo;
+
 class SemaLoongArch : public SemaBase {
 public:
   SemaLoongArch(Sema &S);
diff --git a/clang/include/clang/Sema/SemaM68k.h b/clang/include/clang/Sema/SemaM68k.h
index 5a9767d5ea52163..dece033c22030a9 100644
--- a/clang/include/clang/Sema/SemaM68k.h
+++ b/clang/include/clang/Sema/SemaM68k.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CLANG_SEMA_SEMAM68K_H
 #define LLVM_CLANG_SEMA_SEMAM68K_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
-class Decl;
 class ParsedAttr;
 
 class SemaM68k : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaMIPS.h b/clang/include/clang/Sema/SemaMIPS.h
index 6366dce57626af8..41feeb69b80594b 100644
--- a/clang/include/clang/Sema/SemaMIPS.h
+++ b/clang/include/clang/Sema/SemaMIPS.h
@@ -13,13 +13,12 @@
 #ifndef LLVM_CLANG_SEMA_SEMAMIPS_H
 #define LLVM_CLANG_SEMA_SEMAMIPS_H
 
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
 class ParsedAttr;
+class TargetInfo;
 
 class SemaMIPS : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaMSP430.h b/clang/include/clang/Sema/SemaMSP430.h
index e1034aefe881640..30eb8eaf6aa1a39 100644
--- a/clang/include/clang/Sema/SemaMSP430.h
+++ b/clang/include/clang/Sema/SemaMSP430.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CLANG_SEMA_SEMAMSP430_H
 #define LLVM_CLANG_SEMA_SEMAMSP430_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
-class Decl;
 class ParsedAttr;
 
 class SemaMSP430 : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaNVPTX.h b/clang/include/clang/Sema/SemaNVPTX.h
index a663c694179d98a..de47997d2f8bf16 100644
--- a/clang/include/clang/Sema/SemaNVPTX.h
+++ b/clang/include/clang/Sema/SemaNVPTX.h
@@ -13,11 +13,12 @@
 #ifndef LLVM_CLANG_SEMA_SEMANVPTX_H
 #define LLVM_CLANG_SEMA_SEMANVPTX_H
 
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class TargetInfo;
+
 class SemaNVPTX : public SemaBase {
 public:
   SemaNVPTX(Sema &S);
diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
index 07c3c1a06be1603..213c37b5091fe0f 100644
--- a/clang/include/clang/Sema/SemaObjC.h
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -13,11 +13,8 @@
 #ifndef LLVM_CLANG_SEMA_SEMAOBJC_H
 #define LLVM_CLANG_SEMA_SEMAOBJC_H
 
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclBase.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/DeclObjC.h"
-#include "clang/AST/Expr.h"
-#include "clang/AST/ExprObjC.h"
 #include "clang/AST/NSAPI.h"
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/Type.h"
@@ -27,24 +24,29 @@
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Sema/DeclSpec.h"
-#include "clang/Sema/Lookup.h"
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Redeclaration.h"
-#include "clang/Sema/Scope.h"
+#include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include <memory>
 #include <optional>
-#include <type_traits>
 #include <utility>
 
 namespace clang {
 
+class AttributeCommonInfo;
+class AvailabilitySpec;
 enum class CheckedConversionKind;
+class DeclGroupRef;
+class LookupResult;
+struct ObjCDictionaryElement;
 class ParsedAttr;
+class ParsedAttributesView;
+class Scope;
 struct SkipBodyInfo;
 
 class SemaObjC : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index a5f2a8bf7465777..0ca76842e5f9028 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -16,13 +16,19 @@
 
 #include "clang/AST/DeclGroup.h"
 #include "clang/AST/StmtOpenACC.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/OpenACCKinds.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/SemaBase.h"
+#include "llvm/ADT/SmallVector.h"
+#include <cassert>
+#include <optional>
+#include <utility>
 #include <variant>
 
 namespace clang {
+class IdentifierInfo;
 class OpenACCClause;
 
 class SemaOpenACC : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaOpenCL.h b/clang/include/clang/Sema/SemaOpenCL.h
index 7d6b4b0dec09c25..04b2b617fb12fe6 100644
--- a/clang/include/clang/Sema/SemaOpenCL.h
+++ b/clang/include/clang/Sema/SemaOpenCL.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CLANG_SEMA_SEMAOPENCL_H
 #define LLVM_CLANG_SEMA_SEMAOPENCL_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
-class Decl;
 class ParsedAttr;
 
 class SemaOpenCL : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index e55731212c4a41e..53191e7bb4272bb 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -14,17 +14,12 @@
 #ifndef LLVM_CLANG_SEMA_SEMAOPENMP_H
 #define LLVM_CLANG_SEMA_SEMAOPENMP_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Attr.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DeclarationName.h"
-#include "clang/AST/Expr.h"
 #include "clang/AST/ExprOpenMP.h"
 #include "clang/AST/OpenMPClause.h"
-#include "clang/AST/Stmt.h"
 #include "clang/AST/StmtOpenMP.h"
-#include "clang/AST/Type.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/OpenMPKinds.h"
@@ -32,17 +27,23 @@
 #include "clang/Basic/Specifiers.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Ownership.h"
-#include "clang/Sema/Scope.h"
-#include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/PointerUnion.h"
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include <optional>
 #include <string>
 #include <utility>
 
 namespace clang {
+namespace sema {
+class FunctionScopeInfo;
+} // namespace sema
+
+class DeclContext;
+class DeclGroupRef;
 class ParsedAttr;
+class Scope;
 
 class SemaOpenMP : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaPPC.h b/clang/include/clang/Sema/SemaPPC.h
index 3e8929d5b6dedab..f8edecc4fcb7bff 100644
--- a/clang/include/clang/Sema/SemaPPC.h
+++ b/clang/include/clang/Sema/SemaPPC.h
@@ -13,13 +13,14 @@
 #ifndef LLVM_CLANG_SEMA_SEMAPPC_H
 #define LLVM_CLANG_SEMA_SEMAPPC_H
 
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class TargetInfo;
+
 class SemaPPC : public SemaBase {
 public:
   SemaPPC(Sema &S);
diff --git a/clang/include/clang/Sema/SemaPseudoObject.h b/clang/include/clang/Sema/SemaPseudoObject.h
index 22d8be2b3726ee1..4753987070b750a 100644
--- a/clang/include/clang/Sema/SemaPseudoObject.h
+++ b/clang/include/clang/Sema/SemaPseudoObject.h
@@ -14,14 +14,14 @@
 #ifndef LLVM_CLANG_SEMA_SEMAPSEUDOOBJECT_H
 #define LLVM_CLANG_SEMA_SEMAPSEUDOOBJECT_H
 
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/OperationKinds.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Ownership.h"
-#include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class Scope;
 
 class SemaPseudoObject : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaRISCV.h b/clang/include/clang/Sema/SemaRISCV.h
index 48d15c411bddde2..d62fca8128b2a3a 100644
--- a/clang/include/clang/Sema/SemaRISCV.h
+++ b/clang/include/clang/Sema/SemaRISCV.h
@@ -13,18 +13,21 @@
 #ifndef LLVM_CLANG_SEMA_SEMARISCV_H
 #define LLVM_CLANG_SEMA_SEMARISCV_H
 
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/TargetInfo.h"
-#include "clang/Sema/RISCVIntrinsicManager.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
 #include <memory>
 
 namespace clang {
+namespace sema {
+class RISCVIntrinsicManager;
+} // namespace sema
+
 class ParsedAttr;
+class TargetInfo;
 
 class SemaRISCV : public SemaBase {
 public:
@@ -39,7 +42,7 @@ class SemaRISCV : public SemaBase {
   bool isValidRVVBitcast(QualType srcType, QualType destType);
 
   void handleInterruptAttr(Decl *D, const ParsedAttr &AL);
-  bool isAliasValid(unsigned BuiltinID, StringRef AliasName);
+  bool isAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
 
   /// Indicate RISC-V vector builtin functions enabled or not.
   bool DeclareRVVBuiltins = false;
diff --git a/clang/include/clang/Sema/SemaSYCL.h b/clang/include/clang/Sema/SemaSYCL.h
index 363178546a236c5..27c42b540183073 100644
--- a/clang/include/clang/Sema/SemaSYCL.h
+++ b/clang/include/clang/Sema/SemaSYCL.h
@@ -13,7 +13,7 @@
 #ifndef LLVM_CLANG_SEMA_SEMASYCL_H
 #define LLVM_CLANG_SEMA_SEMASYCL_H
 
-#include "clang/AST/Decl.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Ownership.h"
diff --git a/clang/include/clang/Sema/SemaSwift.h b/clang/include/clang/Sema/SemaSwift.h
index a5561d756affd10..8d8f1467054ac5b 100644
--- a/clang/include/clang/Sema/SemaSwift.h
+++ b/clang/include/clang/Sema/SemaSwift.h
@@ -13,15 +13,15 @@
 #ifndef LLVM_CLANG_SEMA_SEMASWIFT_H
 #define LLVM_CLANG_SEMA_SEMASWIFT_H
 
-#include "clang/AST/Attr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/SemaBase.h"
-#include "llvm/ADT/StringRef.h"
 
 namespace clang {
 class AttributeCommonInfo;
 class Decl;
+enum class ParameterABI;
 class ParsedAttr;
 class SwiftNameAttr;
 
diff --git a/clang/include/clang/Sema/SemaSystemZ.h b/clang/include/clang/Sema/SemaSystemZ.h
index 8945471d53d638f..1a3e29f7a9312a6 100644
--- a/clang/include/clang/Sema/SemaSystemZ.h
+++ b/clang/include/clang/Sema/SemaSystemZ.h
@@ -13,7 +13,7 @@
 #ifndef LLVM_CLANG_SEMA_SEMASYSTEMZ_H
 #define LLVM_CLANG_SEMA_SEMASYSTEMZ_H
 
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
diff --git a/clang/include/clang/Sema/SemaWasm.h b/clang/include/clang/Sema/SemaWasm.h
index c3c781535024aef..8841fdff2303538 100644
--- a/clang/include/clang/Sema/SemaWasm.h
+++ b/clang/include/clang/Sema/SemaWasm.h
@@ -13,14 +13,13 @@
 #ifndef LLVM_CLANG_SEMA_SEMAWASM_H
 #define LLVM_CLANG_SEMA_SEMAWASM_H
 
-#include "clang/AST/Attr.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
-#include "clang/Sema/ParsedAttr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class ParsedAttr;
+class TargetInfo;
+
 class SemaWasm : public SemaBase {
 public:
   SemaWasm(Sema &S);
diff --git a/clang/include/clang/Sema/SemaX86.h b/clang/include/clang/Sema/SemaX86.h
index e53aaf229c38dda..b5a23f1bede047e 100644
--- a/clang/include/clang/Sema/SemaX86.h
+++ b/clang/include/clang/Sema/SemaX86.h
@@ -13,14 +13,13 @@
 #ifndef LLVM_CLANG_SEMA_SEMAX86_H
 #define LLVM_CLANG_SEMA_SEMAX86_H
 
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Basic/LLVM.h"
-#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
 class ParsedAttr;
+class TargetInfo;
 
 class SemaX86 : public SemaBase {
 public:
diff --git a/clang/lib/APINotes/APINotesFormat.h b/clang/lib/APINotes/APINotesFormat.h
index fba5f4e8907dae8..d724a9ea471b54a 100644
--- a/clang/lib/APINotes/APINotesFormat.h
+++ b/clang/lib/APINotes/APINotesFormat.h
@@ -24,7 +24,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// API notes file minor version number.
 ///
 /// When the format changes IN ANY WAY, this number should be incremented.
-const uint16_t VERSION_MINOR = 29; // SwiftConformsTo
+const uint16_t VERSION_MINOR = 30; // fields
 
 const uint8_t kSwiftCopyable = 1;
 const uint8_t kSwiftNonCopyable = 2;
@@ -72,6 +72,10 @@ enum BlockID {
   /// used in other tables.
   OBJC_SELECTOR_BLOCK_ID,
 
+  /// The fields data block, which maps names fields of C records to
+  /// information about the field.
+  FIELD_BLOCK_ID,
+
   /// The global variables data block, which maps global variable names to
   /// information about the global variable.
   GLOBAL_VARIABLE_BLOCK_ID,
@@ -199,6 +203,20 @@ using CXXMethodDataLayout =
                          >;
 } // namespace cxx_method_block
 
+namespace field_block {
+enum {
+  FIELD_DATA = 1,
+};
+
+using FieldDataLayout =
+    llvm::BCRecordLayout<FIELD_DATA,      // record ID
+                         llvm::BCVBR<16>, // table offset within the blob (see
+                                          // below)
+                         llvm::BCBlob     // map from C (context id, name)
+                                          // tuples to C field information
+                         >;
+} // namespace field_block
+
 namespace objc_selector_block {
 enum {
   OBJC_SELECTOR_DATA = 1,
diff --git a/clang/lib/APINotes/APINotesReader.cpp b/clang/lib/APINotes/APINotesReader.cpp
index c05fdffe4a071b8..da8a86f7adb16e0 100644
--- a/clang/lib/APINotes/APINotesReader.cpp
+++ b/clang/lib/APINotes/APINotesReader.cpp
@@ -299,6 +299,28 @@ class ObjCPropertyTableInfo
   }
 };
 
+/// Used to deserialize the on-disk C record field table.
+class FieldTableInfo
+    : public VersionedTableInfo<FieldTableInfo, SingleDeclTableKey, FieldInfo> {
+public:
+  static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
+    auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    return {CtxID, NameID};
+  }
+
+  hash_value_type ComputeHash(internal_key_type Key) {
+    return static_cast<size_t>(Key.hashValue());
+  }
+
+  static FieldInfo readUnversioned(internal_key_type Key,
+                                   const uint8_t *&Data) {
+    FieldInfo Info;
+    ReadVariableInfo(Data, Info);
+    return Info;
+  }
+};
+
 /// Read serialized ParamInfo.
 void ReadParamInfo(const uint8_t *&Data, ParamInfo &Info) {
   ReadVariableInfo(Data, Info);
@@ -653,6 +675,12 @@ class APINotesReader::Implementation {
   /// The Objective-C property table.
   std::unique_ptr<SerializedObjCPropertyTable> ObjCPropertyTable;
 
+  using SerializedFieldTable =
+      llvm::OnDiskIterableChainedHashTable<FieldTableInfo>;
+
+  /// The C record field table.
+  std::unique_ptr<SerializedFieldTable> FieldTable;
+
   using SerializedObjCMethodTable =
       llvm::OnDiskIterableChainedHashTable<ObjCMethodTableInfo>;
 
@@ -720,6 +748,8 @@ class APINotesReader::Implementation {
                            llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readCXXMethodBlock(llvm::BitstreamCursor &Cursor,
                           llvm::SmallVectorImpl<uint64_t> &Scratch);
+  bool readFieldBlock(llvm::BitstreamCursor &Cursor,
+                      llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readObjCSelectorBlock(llvm::BitstreamCursor &Cursor,
                              llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readGlobalVariableBlock(llvm::BitstreamCursor &Cursor,
@@ -1252,6 +1282,81 @@ bool APINotesReader::Implementation::readCXXMethodBlock(
   return false;
 }
 
+bool APINotesReader::Implementation::readFieldBlock(
+    llvm::BitstreamCursor &Cursor, llvm::SmallVectorImpl<uint64_t> &Scratch) {
+  if (Cursor.EnterSubBlock(FIELD_BLOCK_ID))
+    return true;
+
+  llvm::Expected<llvm::BitstreamEntry> MaybeNext = Cursor.advance();
+  if (!MaybeNext) {
+    // FIXME this drops the error on the floor.
+    consumeError(MaybeNext.takeError());
+    return false;
+  }
+  llvm::BitstreamEntry Next = MaybeNext.get();
+  while (Next.Kind != llvm::BitstreamEntry::EndBlock) {
+    if (Next.Kind == llvm::BitstreamEntry::Error)
+      return true;
+
+    if (Next.Kind == llvm::BitstreamEntry::SubBlock) {
+      // Unknown sub-block, possibly for use by a future version of the
+      // API notes format.
+      if (Cursor.SkipBlock())
+        return true;
+
+      MaybeNext = Cursor.advance();
+      if (!MaybeNext) {
+        // FIXME this drops the error on the floor.
+        consumeError(MaybeNext.takeError());
+        return false;
+      }
+      Next = MaybeNext.get();
+      continue;
+    }
+
+    Scratch.clear();
+    llvm::StringRef BlobData;
+    llvm::Expected<unsigned> MaybeKind =
+        Cursor.readRecord(Next.ID, Scratch, &BlobData);
+    if (!MaybeKind) {
+      // FIXME this drops the error on the floor.
+      consumeError(MaybeKind.takeError());
+      return false;
+    }
+    unsigned Kind = MaybeKind.get();
+    switch (Kind) {
+    case field_block::FIELD_DATA: {
+      // Already saw field table.
+      if (FieldTable)
+        return true;
+
+      uint32_t tableOffset;
+      field_block::FieldDataLayout::readRecord(Scratch, tableOffset);
+      auto base = reinterpret_cast<const uint8_t *>(BlobData.data());
+
+      FieldTable.reset(SerializedFieldTable::Create(
+          base + tableOffset, base + sizeof(uint32_t), base));
+      break;
+    }
+
+    default:
+      // Unknown record, possibly for use by a future version of the
+      // module format.
+      break;
+    }
+
+    MaybeNext = Cursor.advance();
+    if (!MaybeNext) {
+      // FIXME this drops the error on the floor.
+      consumeError(MaybeNext.takeError());
+      return false;
+    }
+    Next = MaybeNext.get();
+  }
+
+  return false;
+}
+
 bool APINotesReader::Implementation::readObjCSelectorBlock(
     llvm::BitstreamCursor &Cursor, llvm::SmallVectorImpl<uint64_t> &Scratch) {
   if (Cursor.EnterSubBlock(OBJC_SELECTOR_BLOCK_ID))
@@ -1812,6 +1917,14 @@ APINotesReader::APINotesReader(llvm::MemoryBuffer *InputBuffer,
       }
       break;
 
+    case FIELD_BLOCK_ID:
+      if (!HasValidControlBlock ||
+          Implementation->readFieldBlock(Cursor, Scratch)) {
+        Failed = true;
+        return;
+      }
+      break;
+
     case OBJC_SELECTOR_BLOCK_ID:
       if (!HasValidControlBlock ||
           Implementation->readObjCSelectorBlock(Cursor, Scratch)) {
@@ -2031,6 +2144,23 @@ auto APINotesReader::lookupObjCMethod(ContextID CtxID, ObjCSelectorRef Selector,
   return {Implementation->SwiftVersion, *Known};
 }
 
+auto APINotesReader::lookupField(ContextID CtxID, llvm::StringRef Name)
+    -> VersionedInfo<FieldInfo> {
+  if (!Implementation->FieldTable)
+    return std::nullopt;
+
+  std::optional<IdentifierID> NameID = Implementation->getIdentifier(Name);
+  if (!NameID)
+    return std::nullopt;
+
+  auto Known = Implementation->FieldTable->find(
+      SingleDeclTableKey(CtxID.Value, *NameID));
+  if (Known == Implementation->FieldTable->end())
+    return std::nullopt;
+
+  return {Implementation->SwiftVersion, *Known};
+}
+
 auto APINotesReader::lookupCXXMethod(ContextID CtxID, llvm::StringRef Name)
     -> VersionedInfo<CXXMethodInfo> {
   if (!Implementation->CXXMethodTable)
diff --git a/clang/lib/APINotes/APINotesWriter.cpp b/clang/lib/APINotes/APINotesWriter.cpp
index cf3a0bee393eee4..c452677983bb360 100644
--- a/clang/lib/APINotes/APINotesWriter.cpp
+++ b/clang/lib/APINotes/APINotesWriter.cpp
@@ -62,6 +62,13 @@ class APINotesWriter::Implementation {
       llvm::SmallVector<std::pair<VersionTuple, ObjCPropertyInfo>, 1>>
       ObjCProperties;
 
+  /// Information about C record fields.
+  ///
+  /// Indexed by the context ID and name ID.
+  llvm::DenseMap<SingleDeclTableKey,
+                 llvm::SmallVector<std::pair<VersionTuple, FieldInfo>, 1>>
+      Fields;
+
   /// Information about Objective-C methods.
   ///
   /// Indexed by the context ID, selector ID, and Boolean (stored as a char)
@@ -158,6 +165,7 @@ class APINotesWriter::Implementation {
   void writeObjCPropertyBlock(llvm::BitstreamWriter &Stream);
   void writeObjCMethodBlock(llvm::BitstreamWriter &Stream);
   void writeCXXMethodBlock(llvm::BitstreamWriter &Stream);
+  void writeFieldBlock(llvm::BitstreamWriter &Stream);
   void writeObjCSelectorBlock(llvm::BitstreamWriter &Stream);
   void writeGlobalVariableBlock(llvm::BitstreamWriter &Stream);
   void writeGlobalFunctionBlock(llvm::BitstreamWriter &Stream);
@@ -190,6 +198,7 @@ void APINotesWriter::Implementation::writeToStream(llvm::raw_ostream &OS) {
     writeObjCPropertyBlock(Stream);
     writeObjCMethodBlock(Stream);
     writeCXXMethodBlock(Stream);
+    writeFieldBlock(Stream);
     writeObjCSelectorBlock(Stream);
     writeGlobalVariableBlock(Stream);
     writeGlobalFunctionBlock(Stream);
@@ -858,6 +867,62 @@ void APINotesWriter::Implementation::writeCXXMethodBlock(
   }
 }
 
+namespace {
+/// Used to serialize the on-disk C field table.
+class FieldTableInfo
+    : public VersionedTableInfo<FieldTableInfo, SingleDeclTableKey, FieldInfo> {
+public:
+  unsigned getKeyLength(key_type_ref) {
+    return sizeof(uint32_t) + sizeof(uint32_t);
+  }
+
+  void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
+    writer.write<uint32_t>(Key.parentContextID);
+    writer.write<uint32_t>(Key.nameID);
+  }
+
+  hash_value_type ComputeHash(key_type_ref key) {
+    return static_cast<size_t>(key.hashValue());
+  }
+
+  unsigned getUnversionedInfoSize(const FieldInfo &FI) {
+    return getVariableInfoSize(FI);
+  }
+
+  void emitUnversionedInfo(raw_ostream &OS, const FieldInfo &FI) {
+    emitVariableInfo(OS, FI);
+  }
+};
+} // namespace
+
+void APINotesWriter::Implementation::writeFieldBlock(
+    llvm::BitstreamWriter &Stream) {
+  llvm::BCBlockRAII Scope(Stream, FIELD_BLOCK_ID, 3);
+
+  if (Fields.empty())
+    return;
+
+  {
+    llvm::SmallString<4096> HashTableBlob;
+    uint32_t Offset;
+    {
+      llvm::OnDiskChainedHashTableGenerator<FieldTableInfo> Generator;
+      for (auto &FD : Fields)
+        Generator.insert(FD.first, FD.second);
+
+      llvm::raw_svector_ostream BlobStream(HashTableBlob);
+      // Make sure that no bucket is at offset 0
+      llvm::support::endian::write<uint32_t>(BlobStream, 0,
+                                             llvm::endianness::little);
+      Offset = Generator.Emit(BlobStream);
+    }
+
+    field_block::FieldDataLayout FieldData(Stream);
+    FieldData.emit(Scratch, Offset, HashTableBlob);
+  }
+}
+
 namespace {
 /// Used to serialize the on-disk Objective-C selector table.
 class ObjCSelectorTableInfo {
@@ -1423,6 +1488,14 @@ void APINotesWriter::addCXXMethod(ContextID CtxID, llvm::StringRef Name,
   Implementation->CXXMethods[Key].push_back({SwiftVersion, Info});
 }
 
+void APINotesWriter::addField(ContextID CtxID, llvm::StringRef Name,
+                              const FieldInfo &Info,
+                              VersionTuple SwiftVersion) {
+  IdentifierID NameID = Implementation->getIdentifier(Name);
+  SingleDeclTableKey Key(CtxID.Value, NameID);
+  Implementation->Fields[Key].push_back({SwiftVersion, Info});
+}
+
 void APINotesWriter::addGlobalVariable(std::optional<Context> Ctx,
                                        llvm::StringRef Name,
                                        const GlobalVariableInfo &Info,
diff --git a/clang/lib/APINotes/APINotesYAMLCompiler.cpp b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
index 2205686c4d15c3f..16fd59244086fd4 100644
--- a/clang/lib/APINotes/APINotesYAMLCompiler.cpp
+++ b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
@@ -405,6 +405,38 @@ template <> struct ScalarEnumerationTraits<EnumConvenienceAliasKind> {
 } // namespace yaml
 } // namespace llvm
 
+namespace {
+struct Field {
+  StringRef Name;
+  std::optional<NullabilityKind> Nullability;
+  AvailabilityItem Availability;
+  std::optional<bool> SwiftPrivate;
+  StringRef SwiftName;
+  StringRef Type;
+};
+
+typedef std::vector<Field> FieldsSeq;
+} // namespace
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(Field)
+
+namespace llvm {
+namespace yaml {
+template <> struct MappingTraits<Field> {
+  static void mapping(IO &IO, Field &F) {
+    IO.mapRequired("Name", F.Name);
+    IO.mapOptional("Nullability", F.Nullability, std::nullopt);
+    IO.mapOptional("Availability", F.Availability.Mode,
+                   APIAvailability::Available);
+    IO.mapOptional("AvailabilityMsg", F.Availability.Msg, StringRef(""));
+    IO.mapOptional("SwiftPrivate", F.SwiftPrivate);
+    IO.mapOptional("SwiftName", F.SwiftName, StringRef(""));
+    IO.mapOptional("Type", F.Type, StringRef(""));
+  }
+};
+} // namespace yaml
+} // namespace llvm
+
 namespace {
 struct Tag;
 typedef std::vector<Tag> TagsSeq;
@@ -425,6 +457,7 @@ struct Tag {
   std::optional<EnumConvenienceAliasKind> EnumConvenienceKind;
   std::optional<bool> SwiftCopyable;
   FunctionsSeq Methods;
+  FieldsSeq Fields;
 
   /// Tags that are declared within the current tag. Only the tags that have
   /// corresponding API Notes will be listed.
@@ -463,6 +496,7 @@ template <> struct MappingTraits<Tag> {
     IO.mapOptional("EnumKind", T.EnumConvenienceKind);
     IO.mapOptional("SwiftCopyable", T.SwiftCopyable);
     IO.mapOptional("Methods", T.Methods);
+    IO.mapOptional("Fields", T.Fields);
     IO.mapOptional("Tags", T.Tags);
   }
 };
@@ -793,6 +827,16 @@ class YAMLConverter {
                          SwiftVersion);
   }
 
+  template <typename T>
+  void convertVariable(const T &Entity, VariableInfo &VI) {
+    convertAvailability(Entity.Availability, VI, Entity.Name);
+    VI.setSwiftPrivate(Entity.SwiftPrivate);
+    VI.SwiftName = std::string(Entity.SwiftName);
+    if (Entity.Nullability)
+      VI.setNullabilityAudited(*Entity.Nullability);
+    VI.setType(std::string(Entity.Type));
+  }
+
   void convertContext(std::optional<ContextID> ParentContextID, const Class &C,
                       ContextKind Kind, VersionTuple SwiftVersion) {
     // Write the class.
@@ -848,14 +892,9 @@ class YAMLConverter {
 
       // Translate from Property into ObjCPropertyInfo.
       ObjCPropertyInfo PI;
-      convertAvailability(Property.Availability, PI, Property.Name);
-      PI.setSwiftPrivate(Property.SwiftPrivate);
-      PI.SwiftName = std::string(Property.SwiftName);
-      if (Property.Nullability)
-        PI.setNullabilityAudited(*Property.Nullability);
+      convertVariable(Property, PI);
       if (Property.SwiftImportAsAccessors)
         PI.setSwiftImportAsAccessors(*Property.SwiftImportAsAccessors);
-      PI.setType(std::string(Property.Type));
 
       // Add both instance and class properties with this name.
       if (Property.Kind) {
@@ -970,6 +1009,12 @@ class YAMLConverter {
                                       CI, SwiftVersion);
     Context TagCtx(TagCtxID, ContextKind::Tag);
 
+    for (const auto &Field : T.Fields) {
+      FieldInfo FI;
+      convertVariable(Field, FI);
+      Writer.addField(TagCtxID, Field.Name, FI, SwiftVersion);
+    }
+
     for (const auto &CXXMethod : T.Methods) {
       CXXMethodInfo MI;
       convertFunction(CXXMethod, MI);
@@ -1037,12 +1082,7 @@ class YAMLConverter {
       }
 
       GlobalVariableInfo GVI;
-      convertAvailability(Global.Availability, GVI, Global.Name);
-      GVI.setSwiftPrivate(Global.SwiftPrivate);
-      GVI.SwiftName = std::string(Global.SwiftName);
-      if (Global.Nullability)
-        GVI.setNullabilityAudited(*Global.Nullability);
-      GVI.setType(std::string(Global.Type));
+      convertVariable(Global, GVI);
       Writer.addGlobalVariable(Ctx, Global.Name, GVI, SwiftVersion);
     }
 
diff --git a/clang/lib/ARCMigrate/ARCMT.cpp b/clang/lib/ARCMigrate/ARCMT.cpp
index 5835559bff6b720..1a8a200f2fc1949 100644
--- a/clang/lib/ARCMigrate/ARCMT.cpp
+++ b/clang/lib/ARCMigrate/ARCMT.cpp
@@ -596,7 +596,7 @@ bool MigrationProcess::applyTransform(TransformFn trans,
   for (Rewriter::buffer_iterator
         I = rewriter.buffer_begin(), E = rewriter.buffer_end(); I != E; ++I) {
     FileID FID = I->first;
-    RewriteBuffer &buf = I->second;
+    llvm::RewriteBuffer &buf = I->second;
     OptionalFileEntryRef file =
         Ctx.getSourceManager().getFileEntryRefForID(FID);
     assert(file);
diff --git a/clang/lib/ARCMigrate/ObjCMT.cpp b/clang/lib/ARCMigrate/ObjCMT.cpp
index 4357c8e3f09a550..c1bc7c762088f29 100644
--- a/clang/lib/ARCMigrate/ObjCMT.cpp
+++ b/clang/lib/ARCMigrate/ObjCMT.cpp
@@ -36,6 +36,7 @@
 using namespace clang;
 using namespace arcmt;
 using namespace ento;
+using llvm::RewriteBuffer;
 
 namespace {
 
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 0dbbe0043f70317..b201d201e1ea6ad 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -11,8 +11,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "ByteCode/Context.h"
 #include "CXXABI.h"
-#include "Interp/Context.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTConcept.h"
 #include "clang/AST/ASTMutationListener.h"
@@ -3279,7 +3279,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
 
   case Type::MemberPointer: {
     OS << "M";
-    const auto *MPT = T->getAs<MemberPointerType>();
+    const auto *MPT = T->castAs<MemberPointerType>();
     encodeTypeForFunctionPointerAuth(Ctx, OS, QualType(MPT->getClass(), 0));
     encodeTypeForFunctionPointerAuth(Ctx, OS, MPT->getPointeeType());
     return;
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 198bc34a9f031bb..3bc0a647ebf94ff 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -4429,11 +4429,14 @@ ExpectedDecl ASTNodeImporter::VisitFriendDecl(FriendDecl *D) {
   auto FriendLocOrErr = import(D->getFriendLoc());
   if (!FriendLocOrErr)
     return FriendLocOrErr.takeError();
+  auto EllipsisLocOrErr = import(D->getEllipsisLoc());
+  if (!EllipsisLocOrErr)
+    return EllipsisLocOrErr.takeError();
 
   FriendDecl *FrD;
   if (GetImportedOrCreateDecl(FrD, D, Importer.getToContext(), DC,
-                              *LocationOrErr, ToFU,
-                              *FriendLocOrErr, ToTPLists))
+                              *LocationOrErr, ToFU, *FriendLocOrErr,
+                              *EllipsisLocOrErr, ToTPLists))
     return FrD;
 
   FrD->setAccess(D->getAccess());
diff --git a/clang/lib/AST/Availability.cpp b/clang/lib/AST/Availability.cpp
index 238359a2dedfcf4..cf040fc727d11f4 100644
--- a/clang/lib/AST/Availability.cpp
+++ b/clang/lib/AST/Availability.cpp
@@ -16,33 +16,104 @@
 #include "clang/AST/Decl.h"
 #include "clang/Basic/TargetInfo.h"
 
-namespace clang {
+namespace {
+
+/// Represents the availability of a symbol across platforms.
+struct AvailabilitySet {
+  bool UnconditionallyDeprecated = false;
+  bool UnconditionallyUnavailable = false;
+
+  void insert(clang::AvailabilityInfo &&Availability) {
+    auto *Found = getForPlatform(Availability.Domain);
+    if (Found)
+      Found->mergeWith(std::move(Availability));
+    else
+      Availabilities.emplace_back(std::move(Availability));
+  }
+
+  clang::AvailabilityInfo *getForPlatform(llvm::StringRef Domain) {
+    auto *It = llvm::find_if(Availabilities,
+                             [Domain](const clang::AvailabilityInfo &Info) {
+                               return Domain.compare(Info.Domain) == 0;
+                             });
+    return It == Availabilities.end() ? nullptr : It;
+  }
 
-AvailabilityInfo AvailabilityInfo::createFromDecl(const Decl *Decl) {
-  ASTContext &Context = Decl->getASTContext();
-  StringRef PlatformName = Context.getTargetInfo().getPlatformName();
-  AvailabilityInfo Availability;
+private:
+  llvm::SmallVector<clang::AvailabilityInfo> Availabilities;
+};
 
+static void createInfoForDecl(const clang::Decl *Decl,
+                              AvailabilitySet &Availabilities) {
   // Collect availability attributes from all redeclarations.
   for (const auto *RD : Decl->redecls()) {
-    for (const auto *A : RD->specific_attrs<AvailabilityAttr>()) {
-      if (A->getPlatform()->getName() != PlatformName)
-        continue;
-      Availability = AvailabilityInfo(
+    for (const auto *A : RD->specific_attrs<clang::AvailabilityAttr>()) {
+      Availabilities.insert(clang::AvailabilityInfo(
           A->getPlatform()->getName(), A->getIntroduced(), A->getDeprecated(),
-          A->getObsoleted(), A->getUnavailable(), false, false);
-      break;
+          A->getObsoleted(), A->getUnavailable(), false, false));
     }
 
-    if (const auto *A = RD->getAttr<UnavailableAttr>())
+    if (const auto *A = RD->getAttr<clang::UnavailableAttr>())
       if (!A->isImplicit())
-        Availability.UnconditionallyUnavailable = true;
+        Availabilities.UnconditionallyUnavailable = true;
 
-    if (const auto *A = RD->getAttr<DeprecatedAttr>())
+    if (const auto *A = RD->getAttr<clang::DeprecatedAttr>())
       if (!A->isImplicit())
-        Availability.UnconditionallyDeprecated = true;
+        Availabilities.UnconditionallyDeprecated = true;
   }
-  return Availability;
+}
+
+} // namespace
+
+namespace clang {
+
+void AvailabilityInfo::mergeWith(AvailabilityInfo Other) {
+  if (isDefault() && Other.isDefault())
+    return;
+
+  if (Domain.empty())
+    Domain = Other.Domain;
+
+  UnconditionallyUnavailable |= Other.UnconditionallyUnavailable;
+  UnconditionallyDeprecated |= Other.UnconditionallyDeprecated;
+  Unavailable |= Other.Unavailable;
+
+  Introduced = std::max(Introduced, Other.Introduced);
+
+  // Default VersionTuple is 0.0.0 so if both are non default let's pick the
+  // smallest version number, otherwise select the one that is non-zero if there
+  // is one.
+  if (!Deprecated.empty() && !Other.Deprecated.empty())
+    Deprecated = std::min(Deprecated, Other.Deprecated);
+  else
+    Deprecated = std::max(Deprecated, Other.Deprecated);
+
+  if (!Obsoleted.empty() && !Other.Obsoleted.empty())
+    Obsoleted = std::min(Obsoleted, Other.Obsoleted);
+  else
+    Obsoleted = std::max(Obsoleted, Other.Obsoleted);
+}
+
+AvailabilityInfo AvailabilityInfo::createFromDecl(const Decl *D) {
+  AvailabilitySet Availabilities;
+  // Walk DeclContexts upwards starting from D to find the combined availability
+  // of the symbol.
+  for (const auto *Ctx = D; Ctx;
+       Ctx = llvm::cast_or_null<Decl>(Ctx->getDeclContext()))
+    createInfoForDecl(Ctx, Availabilities);
+
+  if (auto *Avail = Availabilities.getForPlatform(
+          D->getASTContext().getTargetInfo().getPlatformName())) {
+    Avail->UnconditionallyDeprecated = Availabilities.UnconditionallyDeprecated;
+    Avail->UnconditionallyUnavailable =
+        Availabilities.UnconditionallyUnavailable;
+    return std::move(*Avail);
+  }
+
+  AvailabilityInfo Avail;
+  Avail.UnconditionallyDeprecated = Availabilities.UnconditionallyDeprecated;
+  Avail.UnconditionallyUnavailable = Availabilities.UnconditionallyUnavailable;
+  return Avail;
 }
 
 } // namespace clang
diff --git a/clang/lib/AST/Interp/Boolean.h b/clang/lib/AST/ByteCode/Boolean.h
similarity index 95%
rename from clang/lib/AST/Interp/Boolean.h
rename to clang/lib/AST/ByteCode/Boolean.h
index 23f728603676464..f1914ddb9970dc5 100644
--- a/clang/lib/AST/Interp/Boolean.h
+++ b/clang/lib/AST/ByteCode/Boolean.h
@@ -9,25 +9,25 @@
 #ifndef LLVM_CLANG_AST_INTERP_BOOLEAN_H
 #define LLVM_CLANG_AST_INTERP_BOOLEAN_H
 
-#include <cstddef>
-#include <cstdint>
 #include "Integral.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ComparisonCategories.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include <cstddef>
+#include <cstdint>
 
 namespace clang {
 namespace interp {
 
 /// Wrapper around boolean types.
 class Boolean final {
- private:
+private:
   /// Underlying boolean.
   bool V;
 
- public:
+public:
   /// Zero-initializes a boolean.
   Boolean() : V(false) {}
   explicit Boolean(bool V) : V(V) {}
@@ -44,6 +44,7 @@ class Boolean final {
   Boolean operator-() const { return Boolean(V); }
   Boolean operator-(const Boolean &Other) const { return Boolean(V - Other.V); }
   Boolean operator~() const { return Boolean(true); }
+  Boolean operator!() const { return Boolean(!V); }
 
   template <typename Ty, typename = std::enable_if_t<std::is_integral_v<Ty>>>
   explicit operator Ty() const {
@@ -104,8 +105,7 @@ class Boolean final {
 
   static Boolean zero() { return from(false); }
 
-  template <typename T>
-  static Boolean from(T Value, unsigned NumBits) {
+  template <typename T> static Boolean from(T Value, unsigned NumBits) {
     return Boolean(Value);
   }
 
@@ -153,7 +153,7 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Boolean &B) {
   return OS;
 }
 
-}  // namespace interp
-}  // namespace clang
+} // namespace interp
+} // namespace clang
 
 #endif
diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.cpp b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
similarity index 86%
rename from clang/lib/AST/Interp/ByteCodeEmitter.cpp
rename to clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
index a01fa15dc0b7dc9..35ae1547939fdd5 100644
--- a/clang/lib/AST/Interp/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
@@ -195,6 +195,48 @@ Function *ByteCodeEmitter::compileFunc(const FunctionDecl *FuncDecl) {
   return Func;
 }
 
+/// Compile an ObjC block, i.e. ^(){}, that thing.
+///
+/// FIXME: We do not support calling the block though, so we create a function
+/// here but do not compile any code for it.
+Function *ByteCodeEmitter::compileObjCBlock(const BlockExpr *BE) {
+  const BlockDecl *BD = BE->getBlockDecl();
+  // Set up argument indices.
+  unsigned ParamOffset = 0;
+  SmallVector<PrimType, 8> ParamTypes;
+  SmallVector<unsigned, 8> ParamOffsets;
+  llvm::DenseMap<unsigned, Function::ParamDescriptor> ParamDescriptors;
+
+  // Assign descriptors to all parameters.
+  // Composite objects are lowered to pointers.
+  for (const ParmVarDecl *PD : BD->parameters()) {
+    std::optional<PrimType> T = Ctx.classify(PD->getType());
+    PrimType PT = T.value_or(PT_Ptr);
+    Descriptor *Desc = P.createDescriptor(PD, PT);
+    ParamDescriptors.insert({ParamOffset, {PT, Desc}});
+    Params.insert({PD, {ParamOffset, T != std::nullopt}});
+    ParamOffsets.push_back(ParamOffset);
+    ParamOffset += align(primSize(PT));
+    ParamTypes.push_back(PT);
+  }
+
+  if (BD->hasCaptures())
+    return nullptr;
+
+  // Create a handle over the emitted code.
+  Function *Func =
+      P.createFunction(BE, ParamOffset, std::move(ParamTypes),
+                       std::move(ParamDescriptors), std::move(ParamOffsets),
+                       /*HasThisPointer=*/false, /*HasRVO=*/false,
+                       /*IsUnevaluatedBuiltin=*/false);
+
+  assert(Func);
+  Func->setDefined(true);
+  // We don't compile the BlockDecl code at all right now.
+  Func->setIsFullyCompiled(true);
+  return Func;
+}
+
 Scope::Local ByteCodeEmitter::createLocal(Descriptor *D) {
   NextLocalOffset += sizeof(Block);
   unsigned Location = NextLocalOffset;
@@ -206,8 +248,7 @@ void ByteCodeEmitter::emitLabel(LabelTy Label) {
   const size_t Target = Code.size();
   LabelOffsets.insert({Label, Target});
 
-  if (auto It = LabelRelocs.find(Label);
-      It != LabelRelocs.end()) {
+  if (auto It = LabelRelocs.find(Label); It != LabelRelocs.end()) {
     for (unsigned Reloc : It->second) {
       using namespace llvm::support;
 
@@ -228,8 +269,7 @@ int32_t ByteCodeEmitter::getOffset(LabelTy Label) {
   assert(aligned(Position));
 
   // If target is known, compute jump offset.
-  if (auto It = LabelOffsets.find(Label);
-      It != LabelOffsets.end())
+  if (auto It = LabelOffsets.find(Label); It != LabelOffsets.end())
     return It->second - Position;
 
   // Otherwise, record relocation and return dummy offset.
@@ -308,7 +348,8 @@ void emit(Program &P, std::vector<std::byte> &Code, const IntegralAP<true> &Val,
 }
 
 template <typename... Tys>
-bool ByteCodeEmitter::emitOp(Opcode Op, const Tys &... Args, const SourceInfo &SI) {
+bool ByteCodeEmitter::emitOp(Opcode Op, const Tys &...Args,
+                             const SourceInfo &SI) {
   bool Success = true;
 
   // The opcode is followed by arguments. The source info is
diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.h b/clang/lib/AST/ByteCode/ByteCodeEmitter.h
similarity index 96%
rename from clang/lib/AST/Interp/ByteCodeEmitter.h
rename to clang/lib/AST/ByteCode/ByteCodeEmitter.h
index a19a25c2f9e8ecc..7cbbe651699b34b 100644
--- a/clang/lib/AST/Interp/ByteCodeEmitter.h
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.h
@@ -32,6 +32,7 @@ class ByteCodeEmitter {
 public:
   /// Compiles the function into the module.
   Function *compileFunc(const FunctionDecl *FuncDecl);
+  Function *compileObjCBlock(const BlockExpr *BE);
 
 protected:
   ByteCodeEmitter(Context &Ctx, Program &P) : Ctx(Ctx), P(P) {}
@@ -92,7 +93,7 @@ class ByteCodeEmitter {
 
   /// Emits an opcode.
   template <typename... Tys>
-  bool emitOp(Opcode Op, const Tys &... Args, const SourceInfo &L);
+  bool emitOp(Opcode Op, const Tys &...Args, const SourceInfo &L);
 
 protected:
 #define GET_LINK_PROTO
diff --git a/clang/lib/AST/Interp/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
similarity index 99%
rename from clang/lib/AST/Interp/Compiler.cpp
rename to clang/lib/AST/ByteCode/Compiler.cpp
index d32b595f9f07240..131b83ae8eb3976 100644
--- a/clang/lib/AST/Interp/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -327,6 +327,8 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
   case CK_NullToPointer:
   case CK_NullToMemberPointer: {
+    if (!this->discard(SubExpr))
+      return false;
     if (DiscardResult)
       return true;
 
@@ -389,8 +391,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
       return this->emitPop(T, CE);
 
     QualType PtrType = CE->getType();
-    assert(PtrType->isPointerType());
-
     const Descriptor *Desc;
     if (std::optional<PrimType> T = classify(PtrType->getPointeeType()))
       Desc = P.createDescriptor(SubExpr, *T);
@@ -533,10 +533,8 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     // We're creating a complex value here, so we need to
     // allocate storage for it.
     if (!Initializing) {
-      std::optional<unsigned> LocalIndex = allocateLocal(CE);
-      if (!LocalIndex)
-        return false;
-      if (!this->emitGetPtrLocal(*LocalIndex, CE))
+      unsigned LocalIndex = allocateTemporary(CE);
+      if (!this->emitGetPtrLocal(LocalIndex, CE))
         return false;
     }
 
@@ -671,10 +669,8 @@ bool Compiler<Emitter>::VisitImaginaryLiteral(const ImaginaryLiteral *E) {
     return true;
 
   if (!Initializing) {
-    std::optional<unsigned> LocalIndex = allocateLocal(E);
-    if (!LocalIndex)
-      return false;
-    if (!this->emitGetPtrLocal(*LocalIndex, E))
+    unsigned LocalIndex = allocateTemporary(E);
+    if (!this->emitGetPtrLocal(LocalIndex, E))
       return false;
   }
 
@@ -986,10 +982,8 @@ template <class Emitter>
 bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
   // Prepare storage for result.
   if (!Initializing) {
-    std::optional<unsigned> LocalIndex = allocateLocal(E);
-    if (!LocalIndex)
-      return false;
-    if (!this->emitGetPtrLocal(*LocalIndex, E))
+    unsigned LocalIndex = allocateTemporary(E);
+    if (!this->emitGetPtrLocal(LocalIndex, E))
       return false;
   }
 
@@ -1045,10 +1039,7 @@ bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
 
     if (!LHSIsComplex) {
       // This is using the RHS type for the fake-complex LHS.
-      if (auto LHSO = allocateLocal(RHS))
-        LHSOffset = *LHSO;
-      else
-        return false;
+      LHSOffset = allocateTemporary(RHS);
 
       if (!this->emitGetPtrLocal(LHSOffset, E))
         return false;
@@ -1881,15 +1872,26 @@ bool Compiler<Emitter>::VisitAbstractConditionalOperator(
   if (!this->jumpFalse(LabelFalse))
     return false;
 
-  if (!this->delegate(TrueExpr))
-    return false;
+  {
+    LocalScope<Emitter> S(this);
+    if (!this->delegate(TrueExpr))
+      return false;
+    if (!S.destroyLocals())
+      return false;
+  }
+
   if (!this->jump(LabelEnd))
     return false;
 
   this->emitLabel(LabelFalse);
 
-  if (!this->delegate(FalseExpr))
-    return false;
+  {
+    LocalScope<Emitter> S(this);
+    if (!this->delegate(FalseExpr))
+      return false;
+    if (!S.destroyLocals())
+      return false;
+  }
 
   this->fallthrough(LabelEnd);
   this->emitLabel(LabelEnd);
@@ -2238,8 +2240,6 @@ bool Compiler<Emitter>::VisitExprWithCleanups(const ExprWithCleanups *E) {
   LocalScope<Emitter> ES(this);
   const Expr *SubExpr = E->getSubExpr();
 
-  assert(E->getNumObjects() == 0 && "TODO: Implement cleanups");
-
   return this->delegate(SubExpr) && ES.destroyLocals(E);
 }
 
@@ -2909,6 +2909,17 @@ bool Compiler<Emitter>::VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
   return this->emitFree(E->isArrayForm(), E);
 }
 
+template <class Emitter>
+bool Compiler<Emitter>::VisitBlockExpr(const BlockExpr *E) {
+  const Function *Func = nullptr;
+  if (auto F = Compiler<ByteCodeEmitter>(Ctx, P).compileObjCBlock(E))
+    Func = F;
+
+  if (!Func)
+    return false;
+  return this->emitGetFnPtr(Func, E);
+}
+
 template <class Emitter>
 bool Compiler<Emitter>::VisitExpressionTraitExpr(const ExpressionTraitExpr *E) {
   assert(Ctx.getLangOpts().CPlusPlus);
@@ -3549,6 +3560,27 @@ Compiler<Emitter>::allocateLocal(DeclTy &&Src, const ValueDecl *ExtendingDecl) {
   return Local.Offset;
 }
 
+template <class Emitter>
+unsigned Compiler<Emitter>::allocateTemporary(const Expr *E) {
+  QualType Ty = E->getType();
+  assert(!Ty->isRecordType());
+
+  Descriptor *D = P.createDescriptor(
+      E, Ty.getTypePtr(), Descriptor::InlineDescMD, Ty.isConstQualified(),
+      /*IsTemporary=*/true, /*IsMutable=*/false, /*Init=*/nullptr);
+  assert(D);
+
+  Scope::Local Local = this->createLocal(D);
+  VariableScope<Emitter> *S = VarScope;
+  assert(S);
+  // Attach to topmost scope.
+  while (S->getParent())
+    S = S->getParent();
+  assert(S && !S->getParent());
+  S->addLocal(Local);
+  return Local.Offset;
+}
+
 template <class Emitter>
 const RecordType *Compiler<Emitter>::getRecordTy(QualType Ty) {
   if (const PointerType *PT = dyn_cast<PointerType>(Ty))
@@ -3699,7 +3731,8 @@ bool Compiler<Emitter>::visitDeclAndReturn(const VarDecl *VD,
 }
 
 template <class Emitter>
-VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD, bool Toplevel) {
+VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
+                                                 bool Toplevel) {
   // We don't know what to do with these, so just return false.
   if (VD->getType().isNull())
     return false;
@@ -5088,7 +5121,7 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visitBool(SubExpr))
       return false;
 
-    if (!this->emitInvBool(E))
+    if (!this->emitInv(E))
       return false;
 
     if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
@@ -5101,7 +5134,7 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visit(SubExpr))
       return false;
     return DiscardResult ? this->emitPop(*T, E) : this->emitNeg(*T, E);
-  case UO_Plus:                // +x
+  case UO_Plus: // +x
     if (!T)
       return this->emitError(E);
 
@@ -5207,7 +5240,7 @@ bool Compiler<Emitter>::VisitComplexUnaryOperator(const UnaryOperator *E) {
       return false;
     if (!this->emitComplexBoolCast(SubExpr))
       return false;
-    if (!this->emitInvBool(E))
+    if (!this->emitInv(E))
       return false;
     if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
       return this->emitCast(PT_Bool, ET, E);
diff --git a/clang/lib/AST/Interp/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
similarity index 99%
rename from clang/lib/AST/Interp/Compiler.h
rename to clang/lib/AST/ByteCode/Compiler.h
index 112219c49e8bddb..5acfe3c41796c45 100644
--- a/clang/lib/AST/Interp/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -199,6 +199,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   bool VisitStmtExpr(const StmtExpr *E);
   bool VisitCXXNewExpr(const CXXNewExpr *E);
   bool VisitCXXDeleteExpr(const CXXDeleteExpr *E);
+  bool VisitBlockExpr(const BlockExpr *E);
 
   // Statements.
   bool visitCompoundStmt(const CompoundStmt *S);
@@ -297,6 +298,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   /// Allocates a space storing a local given its type.
   std::optional<unsigned>
   allocateLocal(DeclTy &&Decl, const ValueDecl *ExtendingDecl = nullptr);
+  unsigned allocateTemporary(const Expr *E);
 
 private:
   friend class VariableScope<Emitter>;
diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
similarity index 98%
rename from clang/lib/AST/Interp/Context.cpp
rename to clang/lib/AST/ByteCode/Context.cpp
index 92ac28137fdb45d..e682d87b703ad77 100644
--- a/clang/lib/AST/Interp/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -160,6 +160,9 @@ std::optional<PrimType> Context::classify(QualType T) const {
       return PT_Uint16;
     case 8:
       return PT_Uint8;
+    case 1:
+      // Might happen for enum types.
+      return PT_Bool;
     default:
       return PT_IntAP;
     }
@@ -176,7 +179,7 @@ std::optional<PrimType> Context::classify(QualType T) const {
     return PT_MemberPtr;
 
   if (T->isFunctionPointerType() || T->isFunctionReferenceType() ||
-      T->isFunctionType())
+      T->isFunctionType() || T->isBlockPointerType())
     return PT_FnPtr;
 
   if (T->isPointerOrReferenceType() || T->isObjCObjectPointerType())
diff --git a/clang/lib/AST/Interp/Context.h b/clang/lib/AST/ByteCode/Context.h
similarity index 100%
rename from clang/lib/AST/Interp/Context.h
rename to clang/lib/AST/ByteCode/Context.h
diff --git a/clang/lib/AST/Interp/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
similarity index 98%
rename from clang/lib/AST/Interp/Descriptor.cpp
rename to clang/lib/AST/ByteCode/Descriptor.cpp
index 1b7d9f03f8ef0d0..47b8885c9ae3796 100644
--- a/clang/lib/AST/Interp/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -284,6 +284,14 @@ static BlockDtorFn getDtorPrim(PrimType Type) {
 }
 
 static BlockMoveFn getMovePrim(PrimType Type) {
+  if (Type == PT_Float)
+    return moveTy<PrimConv<PT_Float>::T>;
+  if (Type == PT_IntAP)
+    return moveTy<PrimConv<PT_IntAP>::T>;
+  if (Type == PT_IntAPS)
+    return moveTy<PrimConv<PT_IntAPS>::T>;
+  if (Type == PT_MemberPtr)
+    return moveTy<PrimConv<PT_MemberPtr>::T>;
   COMPOSITE_TYPE_SWITCH(Type, return moveTy<T>, return nullptr);
 }
 
diff --git a/clang/lib/AST/Interp/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
similarity index 99%
rename from clang/lib/AST/Interp/Descriptor.h
rename to clang/lib/AST/ByteCode/Descriptor.h
index 6f1adeb898c4307..41899c3bd6831a2 100644
--- a/clang/lib/AST/Interp/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -229,7 +229,7 @@ struct Descriptor final {
   /// Returns the allocated size, including metadata.
   unsigned getAllocSize() const { return AllocSize; }
   /// returns the size of an element when the structure is viewed as an array.
-  unsigned getElemSize()  const { return ElemSize; }
+  unsigned getElemSize() const { return ElemSize; }
   /// Returns the size of the metadata.
   unsigned getMetadataSize() const { return MDSize; }
 
diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Disasm.cpp
rename to clang/lib/AST/ByteCode/Disasm.cpp
diff --git a/clang/lib/AST/Interp/DynamicAllocator.cpp b/clang/lib/AST/ByteCode/DynamicAllocator.cpp
similarity index 100%
rename from clang/lib/AST/Interp/DynamicAllocator.cpp
rename to clang/lib/AST/ByteCode/DynamicAllocator.cpp
diff --git a/clang/lib/AST/Interp/DynamicAllocator.h b/clang/lib/AST/ByteCode/DynamicAllocator.h
similarity index 100%
rename from clang/lib/AST/Interp/DynamicAllocator.h
rename to clang/lib/AST/ByteCode/DynamicAllocator.h
diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp
similarity index 99%
rename from clang/lib/AST/Interp/EvalEmitter.cpp
rename to clang/lib/AST/ByteCode/EvalEmitter.cpp
index 08536536ac3c2eb..e36d86c814e17fd 100644
--- a/clang/lib/AST/Interp/EvalEmitter.cpp
+++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp
@@ -76,9 +76,7 @@ EvaluationResult EvalEmitter::interpretDecl(const VarDecl *VD,
   return std::move(this->EvalResult);
 }
 
-void EvalEmitter::emitLabel(LabelTy Label) {
-  CurrentLabel = Label;
-}
+void EvalEmitter::emitLabel(LabelTy Label) { CurrentLabel = Label; }
 
 EvalEmitter::LabelTy EvalEmitter::getLabel() { return NextLabel++; }
 
diff --git a/clang/lib/AST/Interp/EvalEmitter.h b/clang/lib/AST/ByteCode/EvalEmitter.h
similarity index 100%
rename from clang/lib/AST/Interp/EvalEmitter.h
rename to clang/lib/AST/ByteCode/EvalEmitter.h
diff --git a/clang/lib/AST/Interp/EvaluationResult.cpp b/clang/lib/AST/ByteCode/EvaluationResult.cpp
similarity index 100%
rename from clang/lib/AST/Interp/EvaluationResult.cpp
rename to clang/lib/AST/ByteCode/EvaluationResult.cpp
diff --git a/clang/lib/AST/Interp/EvaluationResult.h b/clang/lib/AST/ByteCode/EvaluationResult.h
similarity index 100%
rename from clang/lib/AST/Interp/EvaluationResult.h
rename to clang/lib/AST/ByteCode/EvaluationResult.h
diff --git a/clang/lib/AST/Interp/Floating.cpp b/clang/lib/AST/ByteCode/Floating.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Floating.cpp
rename to clang/lib/AST/ByteCode/Floating.cpp
diff --git a/clang/lib/AST/Interp/Floating.h b/clang/lib/AST/ByteCode/Floating.h
similarity index 100%
rename from clang/lib/AST/Interp/Floating.h
rename to clang/lib/AST/ByteCode/Floating.h
diff --git a/clang/lib/AST/Interp/Frame.cpp b/clang/lib/AST/ByteCode/Frame.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Frame.cpp
rename to clang/lib/AST/ByteCode/Frame.cpp
diff --git a/clang/lib/AST/Interp/Frame.h b/clang/lib/AST/ByteCode/Frame.h
similarity index 100%
rename from clang/lib/AST/Interp/Frame.h
rename to clang/lib/AST/ByteCode/Frame.h
diff --git a/clang/lib/AST/Interp/Function.cpp b/clang/lib/AST/ByteCode/Function.cpp
similarity index 75%
rename from clang/lib/AST/Interp/Function.cpp
rename to clang/lib/AST/ByteCode/Function.cpp
index 00f5a1fced531a9..e3fab3f6720b41e 100644
--- a/clang/lib/AST/Interp/Function.cpp
+++ b/clang/lib/AST/ByteCode/Function.cpp
@@ -16,16 +16,18 @@
 using namespace clang;
 using namespace clang::interp;
 
-Function::Function(Program &P, const FunctionDecl *F, unsigned ArgSize,
+Function::Function(Program &P, FunctionDeclTy Source, unsigned ArgSize,
                    llvm::SmallVectorImpl<PrimType> &&ParamTypes,
                    llvm::DenseMap<unsigned, ParamDescriptor> &&Params,
                    llvm::SmallVectorImpl<unsigned> &&ParamOffsets,
                    bool HasThisPointer, bool HasRVO, bool UnevaluatedBuiltin)
-    : P(P), Loc(F->getBeginLoc()), F(F), ArgSize(ArgSize),
-      ParamTypes(std::move(ParamTypes)), Params(std::move(Params)),
-      ParamOffsets(std::move(ParamOffsets)), HasThisPointer(HasThisPointer),
-      HasRVO(HasRVO), Variadic(F->isVariadic()),
-      IsUnevaluatedBuiltin(UnevaluatedBuiltin) {}
+    : P(P), Source(Source), ArgSize(ArgSize), ParamTypes(std::move(ParamTypes)),
+      Params(std::move(Params)), ParamOffsets(std::move(ParamOffsets)),
+      HasThisPointer(HasThisPointer), HasRVO(HasRVO),
+      IsUnevaluatedBuiltin(UnevaluatedBuiltin) {
+  if (const auto *F = Source.dyn_cast<const FunctionDecl *>())
+    Variadic = F->isVariadic();
+}
 
 Function::ParamDescriptor Function::getParamDescriptor(unsigned Offset) const {
   auto It = Params.find(Offset);
@@ -46,7 +48,8 @@ SourceInfo Function::getSource(CodePtr PC) const {
 }
 
 bool Function::isVirtual() const {
-  if (const auto *M = dyn_cast<CXXMethodDecl>(F))
+  if (const auto *M = dyn_cast_if_present<CXXMethodDecl>(
+          Source.dyn_cast<const FunctionDecl *>()))
     return M->isVirtual();
   return false;
 }
diff --git a/clang/lib/AST/Interp/Function.h b/clang/lib/AST/ByteCode/Function.h
similarity index 85%
rename from clang/lib/AST/Interp/Function.h
rename to clang/lib/AST/ByteCode/Function.h
index 92bcd96927912db..f254db20d4f5946 100644
--- a/clang/lib/AST/Interp/Function.h
+++ b/clang/lib/AST/ByteCode/Function.h
@@ -20,6 +20,7 @@
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
+#include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
@@ -55,6 +56,9 @@ class Scope final {
   LocalVectorTy Descriptors;
 };
 
+using FunctionDeclTy =
+    llvm::PointerUnion<const FunctionDecl *, const BlockExpr *>;
+
 /// Bytecode function.
 ///
 /// Contains links to the bytecode of the function, as well as metadata
@@ -89,20 +93,22 @@ class Function final {
   CodePtr getCodeEnd() const { return Code.data() + Code.size(); }
 
   /// Returns the original FunctionDecl.
-  const FunctionDecl *getDecl() const { return F; }
+  const FunctionDecl *getDecl() const {
+    return Source.dyn_cast<const FunctionDecl *>();
+  }
+  const BlockExpr *getExpr() const {
+    return Source.dyn_cast<const BlockExpr *>();
+  }
 
   /// Returns the name of the function decl this code
   /// was generated for.
   const std::string getName() const {
-    if (!F)
+    if (!Source)
       return "<<expr>>";
 
-    return F->getQualifiedNameAsString();
+    return Source.get<const FunctionDecl *>()->getQualifiedNameAsString();
   }
 
-  /// Returns the location.
-  SourceLocation getLoc() const { return Loc; }
-
   /// Returns a parameter descriptor.
   ParamDescriptor getParamDescriptor(unsigned Offset) const;
 
@@ -138,13 +144,20 @@ class Function final {
   bool isVirtual() const;
 
   /// Checks if the function is a constructor.
-  bool isConstructor() const { return isa<CXXConstructorDecl>(F); }
+  bool isConstructor() const {
+    return isa_and_nonnull<CXXConstructorDecl>(
+        Source.dyn_cast<const FunctionDecl *>());
+  }
   /// Checks if the function is a destructor.
-  bool isDestructor() const { return isa<CXXDestructorDecl>(F); }
+  bool isDestructor() const {
+    return isa_and_nonnull<CXXDestructorDecl>(
+        Source.dyn_cast<const FunctionDecl *>());
+  }
 
   /// Returns the parent record decl, if any.
   const CXXRecordDecl *getParentDecl() const {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(F))
+    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
+            Source.dyn_cast<const FunctionDecl *>()))
       return MD->getParent();
     return nullptr;
   }
@@ -152,7 +165,8 @@ class Function final {
   /// Returns whether this function is a lambda static invoker,
   /// which we generate custom byte code for.
   bool isLambdaStaticInvoker() const {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(F))
+    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
+            Source.dyn_cast<const FunctionDecl *>()))
       return MD->isLambdaStaticInvoker();
     return false;
   }
@@ -160,7 +174,8 @@ class Function final {
   /// Returns whether this function is the call operator
   /// of a lambda record decl.
   bool isLambdaCallOperator() const {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(F))
+    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
+            Source.dyn_cast<const FunctionDecl *>()))
       return clang::isLambdaCallOperator(MD);
     return false;
   }
@@ -178,9 +193,13 @@ class Function final {
 
   bool isVariadic() const { return Variadic; }
 
-  unsigned getBuiltinID() const { return F->getBuiltinID(); }
+  unsigned getBuiltinID() const {
+    return Source.get<const FunctionDecl *>()->getBuiltinID();
+  }
 
-  bool isBuiltin() const { return F->getBuiltinID() != 0; }
+  bool isBuiltin() const {
+    return Source.get<const FunctionDecl *>()->getBuiltinID() != 0;
+  }
 
   bool isUnevaluatedBuiltin() const { return IsUnevaluatedBuiltin; }
 
@@ -197,7 +216,8 @@ class Function final {
   }
 
   bool isThisPointerExplicit() const {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(F))
+    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
+            Source.dyn_cast<const FunctionDecl *>()))
       return MD->isExplicitObjectMemberFunction();
     return false;
   }
@@ -208,7 +228,7 @@ class Function final {
 
 private:
   /// Construct a function representing an actual function.
-  Function(Program &P, const FunctionDecl *F, unsigned ArgSize,
+  Function(Program &P, FunctionDeclTy Source, unsigned ArgSize,
            llvm::SmallVectorImpl<PrimType> &&ParamTypes,
            llvm::DenseMap<unsigned, ParamDescriptor> &&Params,
            llvm::SmallVectorImpl<unsigned> &&ParamOffsets, bool HasThisPointer,
@@ -235,10 +255,8 @@ class Function final {
 
   /// Program reference.
   Program &P;
-  /// Location of the executed code.
-  SourceLocation Loc;
   /// Declaration this function was compiled from.
-  const FunctionDecl *F;
+  FunctionDeclTy Source;
   /// Local area size: storage + metadata.
   unsigned FrameSize = 0;
   /// Size of the argument stack.
diff --git a/clang/lib/AST/Interp/FunctionPointer.h b/clang/lib/AST/ByteCode/FunctionPointer.h
similarity index 90%
rename from clang/lib/AST/Interp/FunctionPointer.h
rename to clang/lib/AST/ByteCode/FunctionPointer.h
index d92cd32933fcdce..c9bdfbee55441a3 100644
--- a/clang/lib/AST/Interp/FunctionPointer.h
+++ b/clang/lib/AST/ByteCode/FunctionPointer.h
@@ -33,7 +33,7 @@ class FunctionPointer final {
   bool isZero() const { return !Func; }
   bool isValid() const { return Valid; }
   bool isWeak() const {
-    if (!Func || !Valid)
+    if (!Func || !Valid || !Func->getDecl())
       return false;
 
     return Func->getDecl()->isWeak();
@@ -49,7 +49,10 @@ class FunctionPointer final {
                      CharUnits::fromQuantity(getIntegerRepresentation()), {},
                      /*OnePastTheEnd=*/false, /*IsNull=*/false);
 
-    return APValue(Func->getDecl(), CharUnits::Zero(), {},
+    if (Func->getDecl())
+      return APValue(Func->getDecl(), CharUnits::Zero(), {},
+                     /*OnePastTheEnd=*/false, /*IsNull=*/false);
+    return APValue(Func->getExpr(), CharUnits::Zero(), {},
                    /*OnePastTheEnd=*/false, /*IsNull=*/false);
   }
 
diff --git a/clang/lib/AST/Interp/Integral.h b/clang/lib/AST/ByteCode/Integral.h
similarity index 93%
rename from clang/lib/AST/Interp/Integral.h
rename to clang/lib/AST/ByteCode/Integral.h
index aafdd02676c966f..e06ec1669259da4 100644
--- a/clang/lib/AST/Interp/Integral.h
+++ b/clang/lib/AST/ByteCode/Integral.h
@@ -13,8 +13,8 @@
 #ifndef LLVM_CLANG_AST_INTERP_INTEGRAL_H
 #define LLVM_CLANG_AST_INTERP_INTEGRAL_H
 
-#include "clang/AST/ComparisonCategories.h"
 #include "clang/AST/APValue.h"
+#include "clang/AST/ComparisonCategories.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
@@ -33,14 +33,30 @@ template <bool Signed> class IntegralAP;
 
 // Helper structure to select the representation.
 template <unsigned Bits, bool Signed> struct Repr;
-template <> struct Repr<8, false> { using Type = uint8_t; };
-template <> struct Repr<16, false> { using Type = uint16_t; };
-template <> struct Repr<32, false> { using Type = uint32_t; };
-template <> struct Repr<64, false> { using Type = uint64_t; };
-template <> struct Repr<8, true> { using Type = int8_t; };
-template <> struct Repr<16, true> { using Type = int16_t; };
-template <> struct Repr<32, true> { using Type = int32_t; };
-template <> struct Repr<64, true> { using Type = int64_t; };
+template <> struct Repr<8, false> {
+  using Type = uint8_t;
+};
+template <> struct Repr<16, false> {
+  using Type = uint16_t;
+};
+template <> struct Repr<32, false> {
+  using Type = uint32_t;
+};
+template <> struct Repr<64, false> {
+  using Type = uint64_t;
+};
+template <> struct Repr<8, true> {
+  using Type = int8_t;
+};
+template <> struct Repr<16, true> {
+  using Type = int16_t;
+};
+template <> struct Repr<32, true> {
+  using Type = int32_t;
+};
+template <> struct Repr<64, true> {
+  using Type = int64_t;
+};
 
 /// Wrapper around numeric types.
 ///
@@ -159,12 +175,8 @@ template <unsigned Bits, bool Signed> class Integral final {
 
   void print(llvm::raw_ostream &OS) const { OS << V; }
 
-  static Integral min(unsigned NumBits) {
-    return Integral(Min);
-  }
-  static Integral max(unsigned NumBits) {
-    return Integral(Max);
-  }
+  static Integral min(unsigned NumBits) { return Integral(Min); }
+  static Integral max(unsigned NumBits) { return Integral(Max); }
 
   template <typename ValT> static Integral from(ValT Value) {
     if constexpr (std::is_integral<ValT>::value)
diff --git a/clang/lib/AST/Interp/IntegralAP.h b/clang/lib/AST/ByteCode/IntegralAP.h
similarity index 100%
rename from clang/lib/AST/Interp/IntegralAP.h
rename to clang/lib/AST/ByteCode/IntegralAP.h
diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Interp.cpp
rename to clang/lib/AST/ByteCode/Interp.cpp
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/ByteCode/Interp.h
similarity index 98%
rename from clang/lib/AST/Interp/Interp.h
rename to clang/lib/AST/ByteCode/Interp.h
index 3eab0cfd871385a..bad46e983048455 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -656,15 +656,9 @@ inline bool Divf(InterpState &S, CodePtr OpPC, llvm::RoundingMode RM) {
 // Inv
 //===----------------------------------------------------------------------===//
 
-template <PrimType Name, class T = typename PrimConv<Name>::T>
-bool Inv(InterpState &S, CodePtr OpPC) {
-  using BoolT = PrimConv<PT_Bool>::T;
-  const T &Val = S.Stk.pop<T>();
-  const unsigned Bits = Val.bitWidth();
-  Boolean R;
-  Boolean::inv(BoolT::from(Val, Bits), &R);
-
-  S.Stk.push<BoolT>(R);
+inline bool Inv(InterpState &S, CodePtr OpPC) {
+  const auto &Val = S.Stk.pop<Boolean>();
+  S.Stk.push<Boolean>(!Val);
   return true;
 }
 
@@ -1226,7 +1220,7 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool GetField(InterpState &S, CodePtr OpPC, uint32_t I) {
   const Pointer &Obj = S.Stk.peek<Pointer>();
   if (!CheckNull(S, OpPC, Obj, CSK_Field))
-      return false;
+    return false;
   if (!CheckRange(S, OpPC, Obj, CSK_Field))
     return false;
   const Pointer &Field = Obj.atField(I);
@@ -1542,7 +1536,7 @@ inline bool GetPtrActiveField(InterpState &S, CodePtr OpPC, uint32_t Off) {
 }
 
 inline bool GetPtrActiveThisField(InterpState &S, CodePtr OpPC, uint32_t Off) {
- if (S.checkingPotentialConstantExpression())
+  if (S.checkingPotentialConstantExpression())
     return false;
   const Pointer &This = S.Current->getThis();
   if (!CheckThis(S, OpPC, This))
@@ -1853,6 +1847,17 @@ bool OffsetHelper(InterpState &S, CodePtr OpPC, const T &Offset,
   if (!CheckArray(S, OpPC, Ptr))
     return false;
 
+  // This is much simpler for integral pointers, so handle them first.
+  if (Ptr.isIntegralPointer()) {
+    uint64_t V = Ptr.getIntegerRepresentation();
+    uint64_t O = static_cast<uint64_t>(Offset) * Ptr.elemSize();
+    if constexpr (Op == ArithOp::Add)
+      S.Stk.push<Pointer>(V + O, Ptr.asIntPointer().Desc);
+    else
+      S.Stk.push<Pointer>(V - O, Ptr.asIntPointer().Desc);
+    return true;
+  }
+
   uint64_t MaxIndex = static_cast<uint64_t>(Ptr.getNumElems());
   uint64_t Index;
   if (Ptr.isOnePastEnd())
@@ -2053,7 +2058,7 @@ template <PrimType TIn, PrimType TOut> bool Cast(InterpState &S, CodePtr OpPC) {
 /// 1) Pops a Floating from the stack.
 /// 2) Pushes a new floating on the stack that uses the given semantics.
 inline bool CastFP(InterpState &S, CodePtr OpPC, const llvm::fltSemantics *Sem,
-            llvm::RoundingMode RM) {
+                   llvm::RoundingMode RM) {
   Floating F = S.Stk.pop<Floating>();
   Floating Result = F.toSemantics(Sem, RM);
   S.Stk.push<Floating>(Result);
@@ -2313,9 +2318,9 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
     if (!S.noteUndefinedBehavior())
       return false;
     RHS = -RHS;
-    return DoShift < LT, RT,
-           Dir == ShiftDir::Left ? ShiftDir::Right
-                                 : ShiftDir::Left > (S, OpPC, LHS, RHS);
+    return DoShift<LT, RT,
+                   Dir == ShiftDir::Left ? ShiftDir::Right : ShiftDir::Left>(
+        S, OpPC, LHS, RHS);
   }
 
   if constexpr (Dir == ShiftDir::Left) {
@@ -2354,6 +2359,17 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
                                  LT::AsUnsigned::from(RHS, Bits), Bits, &R);
   }
 
+  // We did the shift above as unsigned. Restore the sign bit if we need to.
+  if constexpr (Dir == ShiftDir::Right) {
+    if (LHS.isSigned() && LHS.isNegative()) {
+      typename LT::AsUnsigned SignBit;
+      LT::AsUnsigned::shiftLeft(LT::AsUnsigned::from(1, Bits),
+                                LT::AsUnsigned::from(Bits - 1, Bits), Bits,
+                                &SignBit);
+      LT::AsUnsigned::bitOr(R, SignBit, Bits, &R);
+    }
+  }
+
   S.Stk.push<LT>(LT::from(R));
   return true;
 }
@@ -2467,7 +2483,8 @@ inline bool ArrayElemPop(InterpState &S, CodePtr OpPC, uint32_t Index) {
 }
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
-inline bool CopyArray(InterpState &S, CodePtr OpPC, uint32_t SrcIndex, uint32_t DestIndex, uint32_t Size) {
+inline bool CopyArray(InterpState &S, CodePtr OpPC, uint32_t SrcIndex,
+                      uint32_t DestIndex, uint32_t Size) {
   const auto &SrcPtr = S.Stk.pop<Pointer>();
   const auto &DestPtr = S.Stk.peek<Pointer>();
 
@@ -2702,13 +2719,13 @@ inline bool CallPtr(InterpState &S, CodePtr OpPC, uint32_t ArgSize,
 
   const Function *F = FuncPtr.getFunction();
   if (!F) {
-    const Expr *E = S.Current->getExpr(OpPC);
+    const auto *E = cast<CallExpr>(S.Current->getExpr(OpPC));
     S.FFDiag(E, diag::note_constexpr_null_callee)
-        << const_cast<Expr *>(E) << E->getSourceRange();
+        << const_cast<Expr *>(E->getCallee()) << E->getSourceRange();
     return false;
   }
 
-  if (!FuncPtr.isValid())
+  if (!FuncPtr.isValid() || !F->getDecl())
     return Invalid(S, OpPC);
 
   assert(F);
diff --git a/clang/lib/AST/Interp/InterpBlock.cpp b/clang/lib/AST/ByteCode/InterpBlock.cpp
similarity index 100%
rename from clang/lib/AST/Interp/InterpBlock.cpp
rename to clang/lib/AST/ByteCode/InterpBlock.cpp
diff --git a/clang/lib/AST/Interp/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
similarity index 100%
rename from clang/lib/AST/Interp/InterpBlock.h
rename to clang/lib/AST/ByteCode/InterpBlock.h
index 20cf8cb9d3810e9..a5cd58e3a655a0c 100644
--- a/clang/lib/AST/Interp/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -14,10 +14,10 @@
 #define LLVM_CLANG_AST_INTERP_BLOCK_H
 
 #include "Descriptor.h"
+#include "clang/AST/ComparisonCategories.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/Expr.h"
-#include "clang/AST/ComparisonCategories.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/raw_ostream.h"
 
diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
similarity index 96%
rename from clang/lib/AST/Interp/InterpBuiltin.cpp
rename to clang/lib/AST/ByteCode/InterpBuiltin.cpp
index c3370e2e5286e07..26abf5820510673 100644
--- a/clang/lib/AST/Interp/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -433,6 +433,51 @@ static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
+                                    const InterpFrame *Frame, const Function *F,
+                                    const CallExpr *Call) {
+  const Floating &Arg = S.Stk.peek<Floating>();
+
+  pushInteger(S, Arg.isNegative(), Call->getType());
+  return true;
+}
+
+static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
+                                       const InterpFrame *Frame,
+                                       const Function *F,
+                                       const CallExpr *Call) {
+  const Floating &RHS = S.Stk.peek<Floating>();
+  const Floating &LHS = S.Stk.peek<Floating>(align(2u * primSize(PT_Float)));
+  unsigned ID = F->getBuiltinID();
+
+  pushInteger(
+      S,
+      [&] {
+        switch (ID) {
+        case Builtin::BI__builtin_isgreater:
+          return LHS > RHS;
+        case Builtin::BI__builtin_isgreaterequal:
+          return LHS >= RHS;
+        case Builtin::BI__builtin_isless:
+          return LHS < RHS;
+        case Builtin::BI__builtin_islessequal:
+          return LHS <= RHS;
+        case Builtin::BI__builtin_islessgreater: {
+          ComparisonCategoryResult cmp = LHS.compare(RHS);
+          return cmp == ComparisonCategoryResult::Less ||
+                 cmp == ComparisonCategoryResult::Greater;
+        }
+        case Builtin::BI__builtin_isunordered:
+          return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
+        default:
+          llvm_unreachable("Unexpected builtin ID: Should be a floating point "
+                           "comparison function");
+        }
+      }(),
+      Call->getType());
+  return true;
+}
+
 /// First parameter to __builtin_isfpclass is the floating value, the
 /// second one is an integral value.
 static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
@@ -1313,6 +1358,21 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
     if (!interp__builtin_iszero(S, OpPC, Frame, F, Call))
       return false;
     break;
+  case Builtin::BI__builtin_signbit:
+  case Builtin::BI__builtin_signbitf:
+  case Builtin::BI__builtin_signbitl:
+    if (!interp__builtin_signbit(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+  case Builtin::BI__builtin_isgreater:
+  case Builtin::BI__builtin_isgreaterequal:
+  case Builtin::BI__builtin_isless:
+  case Builtin::BI__builtin_islessequal:
+  case Builtin::BI__builtin_islessgreater:
+  case Builtin::BI__builtin_isunordered:
+    if (!interp_floating_comparison(S, OpPC, Frame, F, Call))
+      return false;
+    break;
   case Builtin::BI__builtin_isfpclass:
     if (!interp__builtin_isfpclass(S, OpPC, Frame, F, Call))
       return false;
diff --git a/clang/lib/AST/Interp/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
similarity index 99%
rename from clang/lib/AST/Interp/InterpFrame.cpp
rename to clang/lib/AST/ByteCode/InterpFrame.cpp
index 4530f28d694e844..8b55b61cbbfa7e5 100644
--- a/clang/lib/AST/Interp/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -137,7 +137,7 @@ void print(llvm::raw_ostream &OS, const Pointer &P, ASTContext &Ctx,
   if (!Ty->isReferenceType())
     OS << "&";
   llvm::SmallVector<Pointer, 2> Levels;
-  for (Pointer F = P; !F.isRoot(); ) {
+  for (Pointer F = P; !F.isRoot();) {
     Levels.push_back(F);
     F = F.isArrayElement() ? F.getArray().expand() : F.getBase();
   }
diff --git a/clang/lib/AST/Interp/InterpFrame.h b/clang/lib/AST/ByteCode/InterpFrame.h
similarity index 99%
rename from clang/lib/AST/Interp/InterpFrame.h
rename to clang/lib/AST/ByteCode/InterpFrame.h
index 91b9b41b5d33431..1e0d2b1d4424b14 100644
--- a/clang/lib/AST/Interp/InterpFrame.h
+++ b/clang/lib/AST/ByteCode/InterpFrame.h
@@ -91,7 +91,7 @@ class InterpFrame final : public Frame {
 
   /// Mutates a local copy of a parameter.
   template <typename T> void setParam(unsigned Offset, const T &Value) {
-     getParamPointer(Offset).deref<T>() = Value;
+    getParamPointer(Offset).deref<T>() = Value;
   }
 
   /// Returns a pointer to an argument - lazily creates a block.
diff --git a/clang/lib/AST/Interp/InterpShared.cpp b/clang/lib/AST/ByteCode/InterpShared.cpp
similarity index 100%
rename from clang/lib/AST/Interp/InterpShared.cpp
rename to clang/lib/AST/ByteCode/InterpShared.cpp
diff --git a/clang/lib/AST/Interp/InterpShared.h b/clang/lib/AST/ByteCode/InterpShared.h
similarity index 100%
rename from clang/lib/AST/Interp/InterpShared.h
rename to clang/lib/AST/ByteCode/InterpShared.h
diff --git a/clang/lib/AST/Interp/InterpStack.cpp b/clang/lib/AST/ByteCode/InterpStack.cpp
similarity index 98%
rename from clang/lib/AST/Interp/InterpStack.cpp
rename to clang/lib/AST/ByteCode/InterpStack.cpp
index c7024740d322ef2..b8cdaeee72166cf 100644
--- a/clang/lib/AST/Interp/InterpStack.cpp
+++ b/clang/lib/AST/ByteCode/InterpStack.cpp
@@ -18,9 +18,7 @@
 using namespace clang;
 using namespace clang::interp;
 
-InterpStack::~InterpStack() {
-  clear();
-}
+InterpStack::~InterpStack() { clear(); }
 
 void InterpStack::clear() {
   if (Chunk && Chunk->Next)
diff --git a/clang/lib/AST/Interp/InterpStack.h b/clang/lib/AST/ByteCode/InterpStack.h
similarity index 97%
rename from clang/lib/AST/Interp/InterpStack.h
rename to clang/lib/AST/ByteCode/InterpStack.h
index 4966e2870de6bd9..153d17f0d70e1bd 100644
--- a/clang/lib/AST/Interp/InterpStack.h
+++ b/clang/lib/AST/ByteCode/InterpStack.h
@@ -32,7 +32,7 @@ class InterpStack final {
   ~InterpStack();
 
   /// Constructs a value in place on the top of the stack.
-  template <typename T, typename... Tys> void push(Tys &&... Args) {
+  template <typename T, typename... Tys> void push(Tys &&...Args) {
     new (grow(aligned_size<T>())) T(std::forward<Tys>(Args)...);
 #ifndef NDEBUG
     ItemTypes.push_back(toPrimType<T>());
@@ -153,8 +153,7 @@ class InterpStack final {
   template <typename T> static constexpr PrimType toPrimType() {
     if constexpr (std::is_same_v<T, Pointer>)
       return PT_Ptr;
-    else if constexpr (std::is_same_v<T, bool> ||
-                       std::is_same_v<T, Boolean>)
+    else if constexpr (std::is_same_v<T, bool> || std::is_same_v<T, Boolean>)
       return PT_Bool;
     else if constexpr (std::is_same_v<T, int8_t> ||
                        std::is_same_v<T, Integral<8, true>>)
diff --git a/clang/lib/AST/Interp/InterpState.cpp b/clang/lib/AST/ByteCode/InterpState.cpp
similarity index 100%
rename from clang/lib/AST/Interp/InterpState.cpp
rename to clang/lib/AST/ByteCode/InterpState.cpp
diff --git a/clang/lib/AST/Interp/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
similarity index 100%
rename from clang/lib/AST/Interp/InterpState.h
rename to clang/lib/AST/ByteCode/InterpState.h
diff --git a/clang/lib/AST/Interp/MemberPointer.cpp b/clang/lib/AST/ByteCode/MemberPointer.cpp
similarity index 64%
rename from clang/lib/AST/Interp/MemberPointer.cpp
rename to clang/lib/AST/ByteCode/MemberPointer.cpp
index 0c1b6edc5f7e105..0fe94db97a3c403 100644
--- a/clang/lib/AST/Interp/MemberPointer.cpp
+++ b/clang/lib/AST/ByteCode/MemberPointer.cpp
@@ -18,8 +18,7 @@ namespace interp {
 std::optional<Pointer> MemberPointer::toPointer(const Context &Ctx) const {
   if (!Dcl || isa<FunctionDecl>(Dcl))
     return Base;
-  const FieldDecl *FD = cast<FieldDecl>(Dcl);
-  assert(FD);
+  assert((isa<FieldDecl, IndirectFieldDecl>(Dcl)));
 
   if (!Base.isBlockPointer())
     return std::nullopt;
@@ -31,24 +30,36 @@ std::optional<Pointer> MemberPointer::toPointer(const Context &Ctx) const {
   if (!BaseRecord)
     return std::nullopt;
 
-  assert(BaseRecord);
-  if (FD->getParent() == BaseRecord->getDecl())
-    return CastedBase.atField(BaseRecord->getField(FD)->Offset);
-
-  const RecordDecl *FieldParent = FD->getParent();
-  const Record *FieldRecord = Ctx.getRecord(FieldParent);
-
   unsigned Offset = 0;
-  Offset += FieldRecord->getField(FD)->Offset;
   Offset += CastedBase.block()->getDescriptor()->getMetadataSize();
 
-  if (Offset > CastedBase.block()->getSize())
-    return std::nullopt;
+  if (const auto *FD = dyn_cast<FieldDecl>(Dcl)) {
+    if (FD->getParent() == BaseRecord->getDecl())
+      return CastedBase.atField(BaseRecord->getField(FD)->Offset);
 
-  if (const RecordDecl *BaseDecl = Base.getDeclPtr().getRecord()->getDecl();
-      BaseDecl != FieldParent)
-    Offset += Ctx.collectBaseOffset(FieldParent, BaseDecl);
+    const RecordDecl *FieldParent = FD->getParent();
+    const Record *FieldRecord = Ctx.getRecord(FieldParent);
 
+    Offset += FieldRecord->getField(FD)->Offset;
+    if (Offset > CastedBase.block()->getSize())
+      return std::nullopt;
+
+    if (const RecordDecl *BaseDecl = Base.getDeclPtr().getRecord()->getDecl();
+        BaseDecl != FieldParent)
+      Offset += Ctx.collectBaseOffset(FieldParent, BaseDecl);
+
+  } else {
+    const auto *IFD = cast<IndirectFieldDecl>(Dcl);
+
+    for (const NamedDecl *ND : IFD->chain()) {
+      const FieldDecl *F = cast<FieldDecl>(ND);
+      const RecordDecl *FieldParent = F->getParent();
+      const Record *FieldRecord = Ctx.getRecord(FieldParent);
+      Offset += FieldRecord->getField(F)->Offset;
+    }
+  }
+
+  assert(BaseRecord);
   if (Offset > CastedBase.block()->getSize())
     return std::nullopt;
 
diff --git a/clang/lib/AST/Interp/MemberPointer.h b/clang/lib/AST/ByteCode/MemberPointer.h
similarity index 100%
rename from clang/lib/AST/Interp/MemberPointer.h
rename to clang/lib/AST/ByteCode/MemberPointer.h
diff --git a/clang/lib/AST/Interp/Opcode.h b/clang/lib/AST/ByteCode/Opcode.h
similarity index 100%
rename from clang/lib/AST/Interp/Opcode.h
rename to clang/lib/AST/ByteCode/Opcode.h
diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
similarity index 99%
rename from clang/lib/AST/Interp/Opcodes.td
rename to clang/lib/AST/ByteCode/Opcodes.td
index 3478eb174e518ed..61319e1633d9ad4 100644
--- a/clang/lib/AST/Interp/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -103,10 +103,6 @@ def PtrTypeClass : TypeClass {
   let Types = [Ptr, FnPtr, MemberPtr];
 }
 
-def BoolTypeClass : TypeClass {
-  let Types = [Bool];
-}
-
 def NonPtrTypeClass : TypeClass {
   let Types = !listconcat(IntegerTypeClass.Types, [Bool], [Float]);
 }
@@ -574,11 +570,8 @@ def Shr : Opcode {
 // Unary operators.
 //===----------------------------------------------------------------------===//
 
-// [Real] -> [Real]
-def Inv: Opcode {
-  let Types = [BoolTypeClass];
-  let HasGroup = 1;
-}
+// [Bool] -> [Bool]
+def Inv: Opcode;
 
 // Increment and decrement.
 def Inc: AluOpcode;
diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
similarity index 97%
rename from clang/lib/AST/Interp/Pointer.cpp
rename to clang/lib/AST/ByteCode/Pointer.cpp
index 466e61666c76e92..5b9e83764cfa503 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -265,10 +265,10 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
 }
 
 void Pointer::print(llvm::raw_ostream &OS) const {
-  OS << PointeeStorage.BS.Pointee << " (";
-  if (isBlockPointer()) {
+  switch (StorageKind) {
+  case Storage::Block: {
     const Block *B = PointeeStorage.BS.Pointee;
-    OS << "Block) {";
+    OS << "(Block) " << B << " {";
 
     if (isRoot())
       OS << "rootptr(" << PointeeStorage.BS.Base << "), ";
@@ -284,11 +284,18 @@ void Pointer::print(llvm::raw_ostream &OS) const {
       OS << B->getSize();
     else
       OS << "nullptr";
-  } else {
-    OS << "Int) {";
-    OS << PointeeStorage.Int.Value << ", " << PointeeStorage.Int.Desc;
+    OS << "}";
+  } break;
+  case Storage::Int:
+    OS << "(Int) {";
+    OS << PointeeStorage.Int.Value << " + " << Offset << ", "
+       << PointeeStorage.Int.Desc;
+    OS << "}";
+    break;
+  case Storage::Fn:
+    OS << "(Fn) { " << asFunctionPointer().getFunction() << " + " << Offset
+       << " }";
   }
-  OS << "}";
 }
 
 std::string Pointer::toDiagnosticString(const ASTContext &Ctx) const {
@@ -647,5 +654,5 @@ IntPointer IntPointer::atOffset(const ASTContext &ASTCtx,
   uint64_t FieldOffset =
       ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex))
           .getQuantity();
-  return IntPointer{this->Desc, FieldOffset};
+  return IntPointer{this->Desc, this->Value + FieldOffset};
 }
diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
similarity index 99%
rename from clang/lib/AST/Interp/Pointer.h
rename to clang/lib/AST/ByteCode/Pointer.h
index 01ccb88ec03b292..ba30449977376bb 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -173,7 +173,7 @@ class Pointer {
 
   /// Subtract the given offset from the current Base and Offset
   /// of the pointer.
-  [[nodiscard]]  Pointer atFieldSub(unsigned Off) const {
+  [[nodiscard]] Pointer atFieldSub(unsigned Off) const {
     assert(Offset >= Off);
     unsigned O = Offset - Off;
     return Pointer(asBlockPointer().Pointee, O, O);
@@ -571,7 +571,7 @@ class Pointer {
   /// Returns the index into an array.
   int64_t getIndex() const {
     if (!isBlockPointer())
-      return 0;
+      return getIntegerRepresentation();
 
     if (isZero())
       return 0;
diff --git a/clang/lib/AST/Interp/PrimType.cpp b/clang/lib/AST/ByteCode/PrimType.cpp
similarity index 100%
rename from clang/lib/AST/Interp/PrimType.cpp
rename to clang/lib/AST/ByteCode/PrimType.cpp
diff --git a/clang/lib/AST/Interp/PrimType.h b/clang/lib/AST/ByteCode/PrimType.h
similarity index 83%
rename from clang/lib/AST/Interp/PrimType.h
rename to clang/lib/AST/ByteCode/PrimType.h
index 20fb5e81774d6ff..bb2f59d86e98d78 100644
--- a/clang/lib/AST/Interp/PrimType.h
+++ b/clang/lib/AST/ByteCode/PrimType.h
@@ -73,23 +73,45 @@ constexpr bool isIntegralType(PrimType T) { return T <= PT_Bool; }
 
 /// Mapping from primitive types to their representation.
 template <PrimType T> struct PrimConv;
-template <> struct PrimConv<PT_Sint8> { using T = Integral<8, true>; };
-template <> struct PrimConv<PT_Uint8> { using T = Integral<8, false>; };
-template <> struct PrimConv<PT_Sint16> { using T = Integral<16, true>; };
-template <> struct PrimConv<PT_Uint16> { using T = Integral<16, false>; };
-template <> struct PrimConv<PT_Sint32> { using T = Integral<32, true>; };
-template <> struct PrimConv<PT_Uint32> { using T = Integral<32, false>; };
-template <> struct PrimConv<PT_Sint64> { using T = Integral<64, true>; };
-template <> struct PrimConv<PT_Uint64> { using T = Integral<64, false>; };
+template <> struct PrimConv<PT_Sint8> {
+  using T = Integral<8, true>;
+};
+template <> struct PrimConv<PT_Uint8> {
+  using T = Integral<8, false>;
+};
+template <> struct PrimConv<PT_Sint16> {
+  using T = Integral<16, true>;
+};
+template <> struct PrimConv<PT_Uint16> {
+  using T = Integral<16, false>;
+};
+template <> struct PrimConv<PT_Sint32> {
+  using T = Integral<32, true>;
+};
+template <> struct PrimConv<PT_Uint32> {
+  using T = Integral<32, false>;
+};
+template <> struct PrimConv<PT_Sint64> {
+  using T = Integral<64, true>;
+};
+template <> struct PrimConv<PT_Uint64> {
+  using T = Integral<64, false>;
+};
 template <> struct PrimConv<PT_IntAP> {
   using T = IntegralAP<false>;
 };
 template <> struct PrimConv<PT_IntAPS> {
   using T = IntegralAP<true>;
 };
-template <> struct PrimConv<PT_Float> { using T = Floating; };
-template <> struct PrimConv<PT_Bool> { using T = Boolean; };
-template <> struct PrimConv<PT_Ptr> { using T = Pointer; };
+template <> struct PrimConv<PT_Float> {
+  using T = Floating;
+};
+template <> struct PrimConv<PT_Bool> {
+  using T = Boolean;
+};
+template <> struct PrimConv<PT_Ptr> {
+  using T = Pointer;
+};
 template <> struct PrimConv<PT_FnPtr> {
   using T = FunctionPointer;
 };
@@ -117,8 +139,12 @@ static inline bool aligned(const void *P) {
 
 /// Helper macro to simplify type switches.
 /// The macro implicitly exposes a type T in the scope of the inner block.
-#define TYPE_SWITCH_CASE(Name, B) \
-  case Name: { using T = PrimConv<Name>::T; B; break; }
+#define TYPE_SWITCH_CASE(Name, B)                                              \
+  case Name: {                                                                 \
+    using T = PrimConv<Name>::T;                                               \
+    B;                                                                         \
+    break;                                                                     \
+  }
 #define TYPE_SWITCH(Expr, B)                                                   \
   do {                                                                         \
     switch (Expr) {                                                            \
@@ -181,7 +207,10 @@ static inline bool aligned(const void *P) {
   do {                                                                         \
     switch (Expr) {                                                            \
       TYPE_SWITCH_CASE(PT_Ptr, B)                                              \
-      default: { D; break; }                                                   \
+    default: {                                                                 \
+      D;                                                                       \
+      break;                                                                   \
+    }                                                                          \
     }                                                                          \
   } while (0)
 #endif
diff --git a/clang/lib/AST/Interp/Primitives.h b/clang/lib/AST/ByteCode/Primitives.h
similarity index 100%
rename from clang/lib/AST/Interp/Primitives.h
rename to clang/lib/AST/ByteCode/Primitives.h
diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
similarity index 95%
rename from clang/lib/AST/Interp/Program.cpp
rename to clang/lib/AST/ByteCode/Program.cpp
index 5dd59d969853cce..a4f0df8bf6462e0 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -76,26 +76,26 @@ unsigned Program::createGlobalString(const StringLiteral *S) {
     Pointer Field = Ptr.atIndex(I).narrow();
     const uint32_t CodePoint = I == N ? 0 : S->getCodeUnit(I);
     switch (CharType) {
-      case PT_Sint8: {
-        using T = PrimConv<PT_Sint8>::T;
-        Field.deref<T>() = T::from(CodePoint, BitWidth);
-        Field.initialize();
-        break;
-      }
-      case PT_Uint16: {
-        using T = PrimConv<PT_Uint16>::T;
-        Field.deref<T>() = T::from(CodePoint, BitWidth);
-        Field.initialize();
-        break;
-      }
-      case PT_Uint32: {
-        using T = PrimConv<PT_Uint32>::T;
-        Field.deref<T>() = T::from(CodePoint, BitWidth);
-        Field.initialize();
-        break;
-      }
-      default:
-        llvm_unreachable("unsupported character type");
+    case PT_Sint8: {
+      using T = PrimConv<PT_Sint8>::T;
+      Field.deref<T>() = T::from(CodePoint, BitWidth);
+      Field.initialize();
+      break;
+    }
+    case PT_Uint16: {
+      using T = PrimConv<PT_Uint16>::T;
+      Field.deref<T>() = T::from(CodePoint, BitWidth);
+      Field.initialize();
+      break;
+    }
+    case PT_Uint32: {
+      using T = PrimConv<PT_Uint32>::T;
+      Field.deref<T>() = T::from(CodePoint, BitWidth);
+      Field.initialize();
+      break;
+    }
+    default:
+      llvm_unreachable("unsupported character type");
     }
   }
   return I;
@@ -329,6 +329,7 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) {
   // Reserve space for fields.
   Record::FieldList Fields;
   for (const FieldDecl *FD : RD->fields()) {
+    FD = FD->getFirstDecl();
     // Note that we DO create fields and descriptors
     // for unnamed bitfields here, even though we later ignore
     // them everywhere. That's so the FieldDecl's getFieldIndex() matches.
@@ -393,8 +394,7 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
             D, ElemTy.getTypePtr(), std::nullopt, IsConst, IsTemporary);
         if (!ElemDesc)
           return nullptr;
-        unsigned ElemSize =
-            ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
+        unsigned ElemSize = ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
         if (std::numeric_limits<unsigned>::max() / ElemSize <= NumElems)
           return {};
         return allocateDescriptor(D, ElemDesc, MDSize, NumElems, IsConst,
diff --git a/clang/lib/AST/Interp/Program.h b/clang/lib/AST/ByteCode/Program.h
similarity index 96%
rename from clang/lib/AST/Interp/Program.h
rename to clang/lib/AST/ByteCode/Program.h
index 1cabc5212180f28..7f69d9790fc79ac 100644
--- a/clang/lib/AST/Interp/Program.h
+++ b/clang/lib/AST/ByteCode/Program.h
@@ -13,8 +13,6 @@
 #ifndef LLVM_CLANG_AST_INTERP_PROGRAM_H
 #define LLVM_CLANG_AST_INTERP_PROGRAM_H
 
-#include <map>
-#include <vector>
 #include "Function.h"
 #include "Pointer.h"
 #include "PrimType.h"
@@ -24,6 +22,8 @@
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Allocator.h"
+#include <map>
+#include <vector>
 
 namespace clang {
 class RecordDecl;
@@ -94,15 +94,14 @@ class Program final {
 
   /// Creates a new function from a code range.
   template <typename... Ts>
-  Function *createFunction(const FunctionDecl *Def, Ts &&... Args) {
+  Function *createFunction(const FunctionDecl *Def, Ts &&...Args) {
     Def = Def->getCanonicalDecl();
     auto *Func = new Function(*this, Def, std::forward<Ts>(Args)...);
     Funcs.insert({Def, std::unique_ptr<Function>(Func)});
     return Func;
   }
   /// Creates an anonymous function.
-  template <typename... Ts>
-  Function *createFunction(Ts &&... Args) {
+  template <typename... Ts> Function *createFunction(Ts &&...Args) {
     auto *Func = new Function(*this, std::forward<Ts>(Args)...);
     AnonFuncs.emplace_back(Func);
     return Func;
@@ -213,8 +212,7 @@ class Program final {
   llvm::DenseMap<const ValueDecl *, unsigned> DummyVariables;
 
   /// Creates a new descriptor.
-  template <typename... Ts>
-  Descriptor *allocateDescriptor(Ts &&... Args) {
+  template <typename... Ts> Descriptor *allocateDescriptor(Ts &&...Args) {
     return new (Allocator) Descriptor(std::forward<Ts>(Args)...);
   }
 
@@ -232,9 +230,7 @@ class Program final {
   }
 
   /// Ends a global declaration.
-  void endDeclaration() {
-    CurrentDeclaration = NoDeclaration;
-  }
+  void endDeclaration() { CurrentDeclaration = NoDeclaration; }
 
 public:
   /// Dumps the disassembled bytecode to \c llvm::errs().
diff --git a/clang/lib/AST/Interp/Record.cpp b/clang/lib/AST/ByteCode/Record.cpp
similarity index 94%
rename from clang/lib/AST/Interp/Record.cpp
rename to clang/lib/AST/ByteCode/Record.cpp
index ac01524e1caf0d7..ec1b55da347af66 100644
--- a/clang/lib/AST/Interp/Record.cpp
+++ b/clang/lib/AST/ByteCode/Record.cpp
@@ -18,7 +18,7 @@ Record::Record(const RecordDecl *Decl, BaseList &&SrcBases,
     : Decl(Decl), Bases(std::move(SrcBases)), Fields(std::move(SrcFields)),
       BaseSize(BaseSize), VirtualSize(VirtualSize), IsUnion(Decl->isUnion()) {
   for (Base &V : SrcVirtualBases)
-    VirtualBases.push_back({ V.Decl, V.Offset + BaseSize, V.Desc, V.R });
+    VirtualBases.push_back({V.Decl, V.Offset + BaseSize, V.Desc, V.R});
 
   for (Base &B : Bases)
     BaseMap[B.Decl] = &B;
@@ -37,7 +37,7 @@ const std::string Record::getName() const {
 }
 
 const Record::Field *Record::getField(const FieldDecl *FD) const {
-  auto It = FieldMap.find(FD);
+  auto It = FieldMap.find(FD->getFirstDecl());
   assert(It != FieldMap.end() && "Missing field");
   return It->second;
 }
diff --git a/clang/lib/AST/Interp/Record.h b/clang/lib/AST/ByteCode/Record.h
similarity index 100%
rename from clang/lib/AST/Interp/Record.h
rename to clang/lib/AST/ByteCode/Record.h
diff --git a/clang/lib/AST/Interp/Source.cpp b/clang/lib/AST/ByteCode/Source.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Source.cpp
rename to clang/lib/AST/ByteCode/Source.cpp
diff --git a/clang/lib/AST/Interp/Source.h b/clang/lib/AST/ByteCode/Source.h
similarity index 100%
rename from clang/lib/AST/Interp/Source.h
rename to clang/lib/AST/ByteCode/Source.h
diff --git a/clang/lib/AST/Interp/State.cpp b/clang/lib/AST/ByteCode/State.cpp
similarity index 100%
rename from clang/lib/AST/Interp/State.cpp
rename to clang/lib/AST/ByteCode/State.cpp
diff --git a/clang/lib/AST/Interp/State.h b/clang/lib/AST/ByteCode/State.h
similarity index 100%
rename from clang/lib/AST/Interp/State.h
rename to clang/lib/AST/ByteCode/State.h
diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt
index 70aecb781c2ff2f..041252b6830e0a7 100644
--- a/clang/lib/AST/CMakeLists.txt
+++ b/clang/lib/AST/CMakeLists.txt
@@ -11,7 +11,7 @@ set(LLVM_LINK_COMPONENTS
 # another library which depends on clangAST.
 clang_tablegen(Opcodes.inc
   -gen-clang-opcodes
-  SOURCE Interp/Opcodes.td
+  SOURCE ByteCode/Opcodes.td
   TARGET Opcodes)
 
 clang_tablegen(AttrDocTable.inc -gen-clang-attr-doc-table
@@ -64,31 +64,31 @@ add_clang_library(clangAST
   ExternalASTSource.cpp
   FormatString.cpp
   InheritViz.cpp
-  Interp/ByteCodeEmitter.cpp
-  Interp/Compiler.cpp
-  Interp/Context.cpp
-  Interp/Descriptor.cpp
-  Interp/Disasm.cpp
-  Interp/EvalEmitter.cpp
-  Interp/Frame.cpp
-  Interp/Function.cpp
-  Interp/InterpBuiltin.cpp
-  Interp/Floating.cpp
-  Interp/EvaluationResult.cpp
-  Interp/DynamicAllocator.cpp
-  Interp/Interp.cpp
-  Interp/InterpBlock.cpp
-  Interp/InterpFrame.cpp
-  Interp/InterpStack.cpp
-  Interp/InterpState.cpp
-  Interp/Pointer.cpp
-  Interp/PrimType.cpp
-  Interp/Program.cpp
-  Interp/Record.cpp
-  Interp/Source.cpp
-  Interp/State.cpp
-  Interp/MemberPointer.cpp
-  Interp/InterpShared.cpp
+  ByteCode/ByteCodeEmitter.cpp
+  ByteCode/Compiler.cpp
+  ByteCode/Context.cpp
+  ByteCode/Descriptor.cpp
+  ByteCode/Disasm.cpp
+  ByteCode/EvalEmitter.cpp
+  ByteCode/Frame.cpp
+  ByteCode/Function.cpp
+  ByteCode/InterpBuiltin.cpp
+  ByteCode/Floating.cpp
+  ByteCode/EvaluationResult.cpp
+  ByteCode/DynamicAllocator.cpp
+  ByteCode/Interp.cpp
+  ByteCode/InterpBlock.cpp
+  ByteCode/InterpFrame.cpp
+  ByteCode/InterpStack.cpp
+  ByteCode/InterpState.cpp
+  ByteCode/Pointer.cpp
+  ByteCode/PrimType.cpp
+  ByteCode/Program.cpp
+  ByteCode/Record.cpp
+  ByteCode/Source.cpp
+  ByteCode/State.cpp
+  ByteCode/MemberPointer.cpp
+  ByteCode/InterpShared.cpp
   ItaniumCXXABI.cpp
   ItaniumMangle.cpp
   JSONNodeDumper.cpp
diff --git a/clang/lib/AST/DeclFriend.cpp b/clang/lib/AST/DeclFriend.cpp
index 04b9b93699f36ce..8b285bfce8d5221 100644
--- a/clang/lib/AST/DeclFriend.cpp
+++ b/clang/lib/AST/DeclFriend.cpp
@@ -31,11 +31,11 @@ FriendDecl *FriendDecl::getNextFriendSlowCase() {
                            NextFriend.get(getASTContext().getExternalSource()));
 }
 
-FriendDecl *FriendDecl::Create(ASTContext &C, DeclContext *DC,
-                               SourceLocation L,
-                               FriendUnion Friend,
-                               SourceLocation FriendL,
-                          ArrayRef<TemplateParameterList *> FriendTypeTPLists) {
+FriendDecl *
+FriendDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L,
+                   FriendUnion Friend, SourceLocation FriendL,
+                   SourceLocation EllipsisLoc,
+                   ArrayRef<TemplateParameterList *> FriendTypeTPLists) {
 #ifndef NDEBUG
   if (Friend.is<NamedDecl *>()) {
     const auto *D = Friend.get<NamedDecl*>();
@@ -56,8 +56,8 @@ FriendDecl *FriendDecl::Create(ASTContext &C, DeclContext *DC,
   std::size_t Extra =
       FriendDecl::additionalSizeToAlloc<TemplateParameterList *>(
           FriendTypeTPLists.size());
-  auto *FD = new (C, DC, Extra) FriendDecl(DC, L, Friend, FriendL,
-                                           FriendTypeTPLists);
+  auto *FD = new (C, DC, Extra)
+      FriendDecl(DC, L, Friend, FriendL, EllipsisLoc, FriendTypeTPLists);
   cast<CXXRecordDecl>(DC)->pushFriendDecl(FD);
   return FD;
 }
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index 26773a69ab9acff..07be813abd8adca 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -868,7 +868,7 @@ void DeclPrinter::VisitFriendDecl(FriendDecl *D) {
     for (unsigned i = 0; i < NumTPLists; ++i)
       printTemplateParameters(D->getFriendTypeTemplateParameterList(i));
     Out << "friend ";
-    Out << " " << TSI->getType().getAsString(Policy);
+    Out << TSI->getType().getAsString(Policy);
   }
   else if (FunctionDecl *FD =
       dyn_cast<FunctionDecl>(D->getFriendDecl())) {
@@ -885,6 +885,9 @@ void DeclPrinter::VisitFriendDecl(FriendDecl *D) {
     Out << "friend ";
     VisitRedeclarableTemplateDecl(CTD);
   }
+
+  if (D->isPackExpansion())
+    Out << "...";
 }
 
 void DeclPrinter::VisitFieldDecl(FieldDecl *D) {
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 57475c66a94e356..9d5b8167d0ee628 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -4759,53 +4759,6 @@ ParenListExpr *ParenListExpr::CreateEmpty(const ASTContext &Ctx,
   return new (Mem) ParenListExpr(EmptyShell(), NumExprs);
 }
 
-/// Certain overflow-dependent code patterns can have their integer overflow
-/// sanitization disabled. Check for the common pattern `if (a + b < a)` and
-/// return the resulting BinaryOperator responsible for the addition so we can
-/// elide overflow checks during codegen.
-static std::optional<BinaryOperator *>
-getOverflowPatternBinOp(const BinaryOperator *E) {
-  Expr *Addition, *ComparedTo;
-  if (E->getOpcode() == BO_LT) {
-    Addition = E->getLHS();
-    ComparedTo = E->getRHS();
-  } else if (E->getOpcode() == BO_GT) {
-    Addition = E->getRHS();
-    ComparedTo = E->getLHS();
-  } else {
-    return {};
-  }
-
-  const Expr *AddLHS = nullptr, *AddRHS = nullptr;
-  BinaryOperator *BO = dyn_cast<BinaryOperator>(Addition);
-
-  if (BO && BO->getOpcode() == clang::BO_Add) {
-    // now store addends for lookup on other side of '>'
-    AddLHS = BO->getLHS();
-    AddRHS = BO->getRHS();
-  }
-
-  if (!AddLHS || !AddRHS)
-    return {};
-
-  const Decl *LHSDecl, *RHSDecl, *OtherDecl;
-
-  LHSDecl = AddLHS->IgnoreParenImpCasts()->getReferencedDeclOfCallee();
-  RHSDecl = AddRHS->IgnoreParenImpCasts()->getReferencedDeclOfCallee();
-  OtherDecl = ComparedTo->IgnoreParenImpCasts()->getReferencedDeclOfCallee();
-
-  if (!OtherDecl)
-    return {};
-
-  if (!LHSDecl && !RHSDecl)
-    return {};
-
-  if ((LHSDecl && LHSDecl == OtherDecl && LHSDecl != RHSDecl) ||
-      (RHSDecl && RHSDecl == OtherDecl && RHSDecl != LHSDecl))
-    return BO;
-  return {};
-}
-
 BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
                                Opcode opc, QualType ResTy, ExprValueKind VK,
                                ExprObjectKind OK, SourceLocation opLoc,
@@ -4815,15 +4768,8 @@ BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
   assert(!isCompoundAssignmentOp() &&
          "Use CompoundAssignOperator for compound assignments");
   BinaryOperatorBits.OpLoc = opLoc;
-  BinaryOperatorBits.ExcludedOverflowPattern = 0;
   SubExprs[LHS] = lhs;
   SubExprs[RHS] = rhs;
-  if (Ctx.getLangOpts().isOverflowPatternExcluded(
-          LangOptions::OverflowPatternExclusionKind::AddOverflowTest)) {
-    std::optional<BinaryOperator *> Result = getOverflowPatternBinOp(this);
-    if (Result.has_value())
-      Result.value()->BinaryOperatorBits.ExcludedOverflowPattern = 1;
-  }
   BinaryOperatorBits.HasFPFeatures = FPFeatures.requiresTrailingStorage();
   if (hasStoredFPFeatures())
     setStoredFPFeatures(FPFeatures);
diff --git a/clang/lib/AST/ExprConstShared.h b/clang/lib/AST/ExprConstShared.h
index 2a7088e4e371a1f..efe8ee986d29b3d 100644
--- a/clang/lib/AST/ExprConstShared.h
+++ b/clang/lib/AST/ExprConstShared.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // Shared functionality between the new constant expression
-// interpreter (AST/Interp/) and the current one (ExprConstant.cpp).
+// interpreter (AST/ByteCode/) and the current one (ExprConstant.cpp).
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 09edbb6641650a0..5540f58b526705a 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -32,10 +32,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "ByteCode/Context.h"
+#include "ByteCode/Frame.h"
+#include "ByteCode/State.h"
 #include "ExprConstShared.h"
-#include "Interp/Context.h"
-#include "Interp/Frame.h"
-#include "Interp/State.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTDiagnostic.h"
@@ -12833,6 +12833,54 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
            Success(Val.isZero() ? 1 : 0, E);
   }
 
+  case Builtin::BI__builtin_signbit:
+  case Builtin::BI__builtin_signbitf:
+  case Builtin::BI__builtin_signbitl: {
+    APFloat Val(0.0);
+    return EvaluateFloat(E->getArg(0), Val, Info) &&
+           Success(Val.isNegative() ? 1 : 0, E);
+  }
+
+  case Builtin::BI__builtin_isgreater:
+  case Builtin::BI__builtin_isgreaterequal:
+  case Builtin::BI__builtin_isless:
+  case Builtin::BI__builtin_islessequal:
+  case Builtin::BI__builtin_islessgreater:
+  case Builtin::BI__builtin_isunordered: {
+    APFloat LHS(0.0);
+    APFloat RHS(0.0);
+    if (!EvaluateFloat(E->getArg(0), LHS, Info) ||
+        !EvaluateFloat(E->getArg(1), RHS, Info))
+      return false;
+
+    return Success(
+        [&] {
+          switch (BuiltinOp) {
+          case Builtin::BI__builtin_isgreater:
+            return LHS > RHS;
+          case Builtin::BI__builtin_isgreaterequal:
+            return LHS >= RHS;
+          case Builtin::BI__builtin_isless:
+            return LHS < RHS;
+          case Builtin::BI__builtin_islessequal:
+            return LHS <= RHS;
+          case Builtin::BI__builtin_islessgreater: {
+            APFloat::cmpResult cmp = LHS.compare(RHS);
+            return cmp == APFloat::cmpResult::cmpLessThan ||
+                   cmp == APFloat::cmpResult::cmpGreaterThan;
+          }
+          case Builtin::BI__builtin_isunordered:
+            return LHS.compare(RHS) == APFloat::cmpResult::cmpUnordered;
+          default:
+            llvm_unreachable("Unexpected builtin ID: Should be a floating "
+                             "point comparison function");
+          }
+        }()
+            ? 1
+            : 0,
+        E);
+  }
+
   case Builtin::BI__builtin_issignaling: {
     APFloat Val(0.0);
     return EvaluateFloat(E->getArg(0), Val, Info) &&
@@ -15567,12 +15615,12 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
         HandleComplexComplexDiv(A, B, C, D, ResR, ResI);
       }
     } else {
-      if (RHS.getComplexIntReal() == 0 && RHS.getComplexIntImag() == 0)
-        return Error(E, diag::note_expr_divide_by_zero);
-
       ComplexValue LHS = Result;
       APSInt Den = RHS.getComplexIntReal() * RHS.getComplexIntReal() +
         RHS.getComplexIntImag() * RHS.getComplexIntImag();
+      if (Den.isZero())
+        return Error(E, diag::note_expr_divide_by_zero);
+
       Result.getComplexIntReal() =
         (LHS.getComplexIntReal() * RHS.getComplexIntReal() +
          LHS.getComplexIntImag() * RHS.getComplexIntImag()) / Den;
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index f8f80c8c2515755..565f1e05710c879 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -1090,6 +1090,7 @@ void JSONNodeDumper::VisitAccessSpecDecl(const AccessSpecDecl *ASD) {
 void JSONNodeDumper::VisitFriendDecl(const FriendDecl *FD) {
   if (const TypeSourceInfo *T = FD->getFriendType())
     JOS.attribute("type", createQualType(T->getType()));
+  attributeOnlyIfTrue("isPackExpansion", FD->isPackExpansion());
 }
 
 void JSONNodeDumper::VisitObjCIvarDecl(const ObjCIvarDecl *D) {
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index db8000e25dc7cc6..ed8d1cf1b98dd80 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -408,8 +408,6 @@ class MicrosoftCXXNameMangler {
   void mangleSourceName(StringRef Name);
   void mangleNestedName(GlobalDecl GD);
 
-  void mangleAutoReturnType(QualType T, QualifierMangleMode QMM);
-
 private:
   bool isStructorDecl(const NamedDecl *ND) const {
     return ND == Structor || getStructor(ND) == Structor;
@@ -479,11 +477,6 @@ class MicrosoftCXXNameMangler {
                           SourceRange Range);
   void mangleObjCKindOfType(const ObjCObjectType *T, Qualifiers Quals,
                             SourceRange Range);
-
-  void mangleAutoReturnType(const MemberPointerType *T, Qualifiers Quals);
-  void mangleAutoReturnType(const PointerType *T, Qualifiers Quals);
-  void mangleAutoReturnType(const LValueReferenceType *T, Qualifiers Quals);
-  void mangleAutoReturnType(const RValueReferenceType *T, Qualifiers Quals);
 };
 }
 
@@ -2501,57 +2494,6 @@ void MicrosoftCXXNameMangler::mangleAddressSpaceType(QualType T,
   mangleArtificialTagType(TagTypeKind::Struct, ASMangling, {"__clang"});
 }
 
-void MicrosoftCXXNameMangler::mangleAutoReturnType(QualType T,
-                                                   QualifierMangleMode QMM) {
-  assert(getASTContext().getLangOpts().isCompatibleWithMSVC(
-             LangOptions::MSVC2019) &&
-         "Cannot mangle MSVC 2017 auto return types!");
-
-  if (isa<AutoType>(T)) {
-    const auto *AT = T->getContainedAutoType();
-    Qualifiers Quals = T.getLocalQualifiers();
-
-    if (QMM == QMM_Result)
-      Out << '?';
-    if (QMM != QMM_Drop)
-      mangleQualifiers(Quals, false);
-    Out << (AT->isDecltypeAuto() ? "_T" : "_P");
-    return;
-  }
-
-  T = T.getDesugaredType(getASTContext());
-  Qualifiers Quals = T.getLocalQualifiers();
-
-  switch (QMM) {
-  case QMM_Drop:
-  case QMM_Result:
-    break;
-  case QMM_Mangle:
-    mangleQualifiers(Quals, false);
-    break;
-  default:
-    llvm_unreachable("QMM_Escape unexpected");
-  }
-
-  const Type *ty = T.getTypePtr();
-  switch (ty->getTypeClass()) {
-  case Type::MemberPointer:
-    mangleAutoReturnType(cast<MemberPointerType>(ty), Quals);
-    break;
-  case Type::Pointer:
-    mangleAutoReturnType(cast<PointerType>(ty), Quals);
-    break;
-  case Type::LValueReference:
-    mangleAutoReturnType(cast<LValueReferenceType>(ty), Quals);
-    break;
-  case Type::RValueReference:
-    mangleAutoReturnType(cast<RValueReferenceType>(ty), Quals);
-    break;
-  default:
-    llvm_unreachable("Invalid type expected");
-  }
-}
-
 void MicrosoftCXXNameMangler::mangleType(QualType T, SourceRange Range,
                                          QualifierMangleMode QMM) {
   // Don't use the canonical types.  MSVC includes things like 'const' on
@@ -2965,52 +2907,17 @@ void MicrosoftCXXNameMangler::mangleFunctionType(const FunctionType *T,
       // can differ by their calling convention and are typically deduced.  So
       // we make sure that this type gets mangled properly.
       mangleType(ResultType, Range, QMM_Result);
-    } else if (IsInLambda) {
-      if (const auto *AT = ResultType->getContainedAutoType()) {
-        assert(AT->getKeyword() == AutoTypeKeyword::Auto &&
-               "should only need to mangle auto!");
-        (void)AT;
-        Out << '?';
-        mangleQualifiers(ResultType.getLocalQualifiers(), /*IsMember=*/false);
-        Out << '?';
-        mangleSourceName("<auto>");
-        Out << '@';
-      } else {
-        Out << '@';
-      }
-    } else if (const auto *AT = ResultType->getContainedAutoType()) {
+    } else if (const auto *AT = dyn_cast_or_null<AutoType>(
+                   ResultType->getContainedAutoType())) {
+      Out << '?';
+      mangleQualifiers(ResultType.getLocalQualifiers(), /*IsMember=*/false);
+      Out << '?';
       assert(AT->getKeyword() != AutoTypeKeyword::GNUAutoType &&
              "shouldn't need to mangle __auto_type!");
-
-      // If we have any pointer types with the clang address space extension
-      // then defer to the custom clang mangling to keep backwards
-      // compatibility. See `mangleType(const PointerType *T, Qualifiers Quals,
-      // SourceRange Range)` for details.
-      auto UseClangMangling = [](QualType ResultType) {
-        QualType T = ResultType;
-        while (isa<PointerType>(T.getTypePtr())) {
-          T = T->getPointeeType();
-          if (T.getQualifiers().hasAddressSpace())
-            return true;
-        }
-        return false;
-      };
-
-      if (getASTContext().getLangOpts().isCompatibleWithMSVC(
-              LangOptions::MSVC2019) &&
-          !UseClangMangling(ResultType)) {
-        if (D && !D->getPrimaryTemplate()) {
-          Out << '@';
-        } else {
-          mangleAutoReturnType(ResultType, QMM_Result);
-        }
-      } else {
-        Out << '?';
-        mangleQualifiers(ResultType.getLocalQualifiers(), /*IsMember=*/false);
-        Out << '?';
-        mangleSourceName(AT->isDecltypeAuto() ? "<decltype-auto>" : "<auto>");
-        Out << '@';
-      }
+      mangleSourceName(AT->isDecltypeAuto() ? "<decltype-auto>" : "<auto>");
+      Out << '@';
+    } else if (IsInLambda) {
+      Out << '@';
     } else {
       if (ResultType->isVoidType())
         ResultType = ResultType.getUnqualifiedType();
@@ -4313,57 +4220,6 @@ void MicrosoftMangleContextImpl::mangleStringLiteral(const StringLiteral *SL,
   Mangler.getStream() << '@';
 }
 
-void MicrosoftCXXNameMangler::mangleAutoReturnType(const MemberPointerType *T,
-                                                   Qualifiers Quals) {
-  QualType PointeeType = T->getPointeeType();
-  manglePointerCVQualifiers(Quals);
-  manglePointerExtQualifiers(Quals, PointeeType);
-  if (const FunctionProtoType *FPT = PointeeType->getAs<FunctionProtoType>()) {
-    Out << '8';
-    mangleName(T->getClass()->castAs<RecordType>()->getDecl());
-    mangleFunctionType(FPT, nullptr, true);
-  } else {
-    mangleQualifiers(PointeeType.getQualifiers(), true);
-    mangleName(T->getClass()->castAs<RecordType>()->getDecl());
-    mangleAutoReturnType(PointeeType, QMM_Drop);
-  }
-}
-
-void MicrosoftCXXNameMangler::mangleAutoReturnType(const PointerType *T,
-                                                   Qualifiers Quals) {
-  QualType PointeeType = T->getPointeeType();
-  assert(!PointeeType.getQualifiers().hasAddressSpace() &&
-         "Unexpected address space mangling required");
-
-  manglePointerCVQualifiers(Quals);
-  manglePointerExtQualifiers(Quals, PointeeType);
-
-  if (const FunctionProtoType *FPT = PointeeType->getAs<FunctionProtoType>()) {
-    Out << '6';
-    mangleFunctionType(FPT);
-  } else {
-    mangleAutoReturnType(PointeeType, QMM_Mangle);
-  }
-}
-
-void MicrosoftCXXNameMangler::mangleAutoReturnType(const LValueReferenceType *T,
-                                                   Qualifiers Quals) {
-  QualType PointeeType = T->getPointeeType();
-  assert(!Quals.hasConst() && !Quals.hasVolatile() && "unexpected qualifier!");
-  Out << 'A';
-  manglePointerExtQualifiers(Quals, PointeeType);
-  mangleAutoReturnType(PointeeType, QMM_Mangle);
-}
-
-void MicrosoftCXXNameMangler::mangleAutoReturnType(const RValueReferenceType *T,
-                                                   Qualifiers Quals) {
-  QualType PointeeType = T->getPointeeType();
-  assert(!Quals.hasConst() && !Quals.hasVolatile() && "unexpected qualifier!");
-  Out << "$$Q";
-  manglePointerExtQualifiers(Quals, PointeeType);
-  mangleAutoReturnType(PointeeType, QMM_Mangle);
-}
-
 MicrosoftMangleContext *MicrosoftMangleContext::create(ASTContext &Context,
                                                        DiagnosticsEngine &Diags,
                                                        bool IsAux) {
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index fbfe92318dc5eee..b748093831e3f50 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -461,6 +461,7 @@ class ODRDeclVisitor : public ConstDeclVisitor<ODRDeclVisitor> {
     } else {
       AddDecl(D->getFriendDecl());
     }
+    Hash.AddBoolean(D->isPackExpansion());
   }
 
   void VisitTemplateTypeParmDecl(const TemplateTypeParmDecl *D) {
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index d50d4c7028c6972..2c962253c8bea42 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -2697,6 +2697,8 @@ void TextNodeDumper::VisitAccessSpecDecl(const AccessSpecDecl *D) {
 void TextNodeDumper::VisitFriendDecl(const FriendDecl *D) {
   if (TypeSourceInfo *T = D->getFriendType())
     dumpType(T->getType());
+  if (D->isPackExpansion())
+    OS << "...";
 }
 
 void TextNodeDumper::VisitObjCIvarDecl(const ObjCIvarDecl *D) {
diff --git a/clang/lib/Analysis/FlowSensitive/ASTOps.cpp b/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
index 27d42a7b508562e..fdba139628d8ff0 100644
--- a/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
+++ b/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
@@ -170,6 +170,13 @@ static void insertIfGlobal(const Decl &D,
       Globals.insert(V);
 }
 
+static void insertIfLocal(const Decl &D,
+                          llvm::DenseSet<const VarDecl *> &Locals) {
+  if (auto *V = dyn_cast<VarDecl>(&D))
+    if (V->hasLocalStorage() && !isa<ParmVarDecl>(V))
+      Locals.insert(V);
+}
+
 static void insertIfFunction(const Decl &D,
                              llvm::DenseSet<const FunctionDecl *> &Funcs) {
   if (auto *FD = dyn_cast<FunctionDecl>(&D))
@@ -220,12 +227,14 @@ class ReferencedDeclsVisitor
 
   bool VisitDecl(Decl *D) {
     insertIfGlobal(*D, Referenced.Globals);
+    insertIfLocal(*D, Referenced.Locals);
     insertIfFunction(*D, Referenced.Functions);
     return true;
   }
 
   bool VisitDeclRefExpr(DeclRefExpr *E) {
     insertIfGlobal(*E->getDecl(), Referenced.Globals);
+    insertIfLocal(*E->getDecl(), Referenced.Locals);
     insertIfFunction(*E->getDecl(), Referenced.Functions);
     return true;
   }
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 051381edabf0b25..da7446913f7c87e 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -402,9 +402,9 @@ AST_MATCHER(CXXConstructExpr, isSafeSpanTwoParamConstruct) {
 
   QualType Arg0Ty = Arg0->IgnoreImplicit()->getType();
 
-  if (Arg0Ty->isConstantArrayType()) {
-    const APSInt ConstArrSize =
-        APSInt(cast<ConstantArrayType>(Arg0Ty)->getSize());
+  if (auto *ConstArrTy =
+          Finder->getASTContext().getAsConstantArrayType(Arg0Ty)) {
+    const APSInt ConstArrSize = APSInt(ConstArrTy->getSize());
 
     // Check form 4:
     return Arg1CV && APSInt::compareValues(ConstArrSize, *Arg1CV) == 0;
@@ -2664,7 +2664,7 @@ static FixItList fixVarDeclWithArray(const VarDecl *D, const ASTContext &Ctx,
 
   // Note: the code below expects the declaration to not use any type sugar like
   // typedef.
-  if (auto CAT = dyn_cast<clang::ConstantArrayType>(D->getType())) {
+  if (auto CAT = Ctx.getAsConstantArrayType(D->getType())) {
     const QualType &ArrayEltT = CAT->getElementType();
     assert(!ArrayEltT.isNull() && "Trying to fix a non-array type variable!");
     // FIXME: support multi-dimensional arrays
@@ -2797,8 +2797,7 @@ fixVariable(const VarDecl *VD, FixitStrategy::Kind K,
     return {};
   }
   case FixitStrategy::Kind::Array: {
-    if (VD->isLocalVarDecl() &&
-        isa<clang::ConstantArrayType>(VD->getType().getCanonicalType()))
+    if (VD->isLocalVarDecl() && Ctx.getAsConstantArrayType(VD->getType()))
       return fixVariableWithArray(VD, Tracker, Ctx, Handler);
 
     DEBUG_NOTE_DECL_FAIL(VD, " : not a local const-size array");
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 533a9fe88a21508..b0256a8ce9ed042 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2038,8 +2038,7 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
   std::pair<bool, bool> InSameTU = isInTheSameTranslationUnit(LOffs, ROffs);
   if (InSameTU.first)
     return InSameTU.second;
-  // TODO: This should be unreachable, but some clients are calling this
-  //       function before making sure LHS and RHS are in the same TU.
+  // This case is used by libclang: clang_isBeforeInTranslationUnit
   return LOffs.first < ROffs.first;
 }
 
diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp
index 35e8f79ba1bac7b..be91b85e3a816f8 100644
--- a/clang/lib/CodeGen/ABIInfoImpl.cpp
+++ b/clang/lib/CodeGen/ABIInfoImpl.cpp
@@ -38,8 +38,9 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
                                 : Context.LongLongTy))
       return getNaturalAlignIndirect(Ty);
 
-  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
-                                            : ABIArgInfo::getDirect());
+  return (isPromotableIntegerTypeForABI(Ty)
+              ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
+              : ABIArgInfo::getDirect());
 }
 
 ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 81e6702d5de6661..34c08818dbb9ade 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -471,7 +471,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
   Options.MCOptions.Dwarf64 = CodeGenOpts.Dwarf64;
   Options.MCOptions.PreserveAsmComments = CodeGenOpts.PreserveAsmComments;
   Options.MCOptions.Crel = CodeGenOpts.Crel;
-  Options.MCOptions.X86RelaxRelocations = CodeGenOpts.RelaxELFRelocations;
+  Options.MCOptions.X86RelaxRelocations = CodeGenOpts.X86RelaxRelocations;
   Options.MCOptions.CompressDebugSections =
       CodeGenOpts.getCompressDebugSections();
   Options.MCOptions.ABIName = TargetOpts.ABI;
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 1c0baeaee036321..f424ddaa175400c 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -58,6 +58,7 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/MatrixBuilder.h"
 #include "llvm/IR/MemoryModelRelaxationAnnotations.h"
+#include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ScopedPrinter.h"
@@ -18919,8 +18920,6 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrin, { Src0->getType() });
     return Builder.CreateCall(F, { Src0, Builder.getFalse() });
   }
-  case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
-  case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
   case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
   case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
   case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
@@ -18932,18 +18931,11 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     Intrinsic::ID IID;
     llvm::Type *ArgTy = llvm::Type::getDoubleTy(getLLVMContext());
     switch (BuiltinID) {
-    case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
-      ArgTy = llvm::Type::getFloatTy(getLLVMContext());
-      IID = Intrinsic::amdgcn_global_atomic_fadd;
-      break;
     case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
       ArgTy = llvm::FixedVectorType::get(
           llvm::Type::getHalfTy(getLLVMContext()), 2);
       IID = Intrinsic::amdgcn_global_atomic_fadd;
       break;
-    case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
-      IID = Intrinsic::amdgcn_global_atomic_fadd;
-      break;
     case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
       IID = Intrinsic::amdgcn_global_atomic_fmin;
       break;
@@ -19366,7 +19358,9 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
   case AMDGPU::BI__builtin_amdgcn_ds_faddf:
   case AMDGPU::BI__builtin_amdgcn_ds_fminf:
-  case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
+  case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
+  case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
+  case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64: {
     llvm::AtomicRMWInst::BinOp BinOp;
     switch (BuiltinID) {
     case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
@@ -19382,6 +19376,8 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
     case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
     case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
+    case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
+    case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
       BinOp = llvm::AtomicRMWInst::FAdd;
       break;
     case AMDGPU::BI__builtin_amdgcn_ds_fminf:
@@ -19416,9 +19412,14 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
       ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(2)),
                               EmitScalarExpr(E->getArg(3)), AO, SSID);
     } else {
-      // The ds_atomic_fadd_* builtins do not have syncscope/order arguments.
-      SSID = llvm::SyncScope::System;
-      AO = AtomicOrdering::SequentiallyConsistent;
+      // Most of the builtins do not have syncscope/order arguments. For DS
+      // atomics the scope doesn't really matter, as they implicitly operate at
+      // workgroup scope.
+      //
+      // The global/flat cases need to use agent scope to consistently produce
+      // the native instruction instead of a cmpxchg expansion.
+      SSID = getLLVMContext().getOrInsertSyncScopeID("agent");
+      AO = AtomicOrdering::Monotonic;
 
       // The v2bf16 builtin uses i16 instead of a natural bfloat type.
       if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16) {
@@ -19432,6 +19433,20 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
         Builder.CreateAtomicRMW(BinOp, Ptr, Val, AO, SSID);
     if (Volatile)
       RMW->setVolatile(true);
+
+    unsigned AddrSpace = Ptr.getType()->getAddressSpace();
+    if (AddrSpace != llvm::AMDGPUAS::LOCAL_ADDRESS) {
+      // Most targets require "amdgpu.no.fine.grained.memory" to emit the native
+      // instruction for flat and global operations.
+      llvm::MDTuple *EmptyMD = MDNode::get(getLLVMContext(), {});
+      RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
+
+      // Most targets require "amdgpu.ignore.denormal.mode" to emit the native
+      // instruction, but this only matters for float fadd.
+      if (BinOp == llvm::AtomicRMWInst::FAdd && Val->getType()->isFloatTy())
+        RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
+    }
+
     return Builder.CreateBitCast(RMW, OrigTy);
   }
   case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 0672861790633bb..48d9a3b8a5acb3b 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -42,6 +42,7 @@
 #include "llvm/IR/MatrixBuilder.h"
 #include "llvm/Passes/OptimizationLevel.h"
 #include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/Endian.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SaveAndRestore.h"
@@ -65,6 +66,22 @@ static llvm::cl::opt<bool> ClSanitizeGuardChecks(
     "ubsan-guard-checks", llvm::cl::Optional,
     llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`."));
 
+//===--------------------------------------------------------------------===//
+//                        Defines for metadata
+//===--------------------------------------------------------------------===//
+
+// Those values are crucial to be the SAME as in ubsan runtime library.
+enum VariableTypeDescriptorKind : uint16_t {
+  /// An integer type.
+  TK_Integer = 0x0000,
+  /// A floating-point type.
+  TK_Float = 0x0001,
+  /// An _BitInt(N) type.
+  TK_BitInt = 0x0002,
+  /// Any other type. The value representation is unspecified.
+  TK_Unknown = 0xffff
+};
+
 //===--------------------------------------------------------------------===//
 //                        Miscellaneous Helper Methods
 //===--------------------------------------------------------------------===//
@@ -3288,22 +3305,40 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
 ///   { i16 TypeKind, i16 TypeInfo }
 /// \endcode
 ///
-/// followed by an array of i8 containing the type name. TypeKind is 0 for an
-/// integer, 1 for a floating point value, and -1 for anything else.
+/// followed by an array of i8 containing the type name with extra information
+/// for BitInt. TypeKind is TK_Integer(0) for an integer, TK_Float(1) for a
+/// floating point value, TK_BitInt(2) for BitInt and TK_Unknown(0xFFFF) for
+/// anything else.
 llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
   // Only emit each type's descriptor once.
   if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))
     return C;
 
-  uint16_t TypeKind = -1;
+  uint16_t TypeKind = TK_Unknown;
   uint16_t TypeInfo = 0;
+  bool IsBitInt = false;
 
   if (T->isIntegerType()) {
-    TypeKind = 0;
+    TypeKind = TK_Integer;
     TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
                (T->isSignedIntegerType() ? 1 : 0);
+    // Follow suggestion from discussion of issue 64100.
+    // So we can write the exact amount of bits in TypeName after '\0'
+    // making it <diagnostic-like type name>.'\0'.<32-bit width>.
+    if (T->isSignedIntegerType() && T->getAs<BitIntType>()) {
+      // Do a sanity checks as we are using 32-bit type to store bit length.
+      assert(getContext().getTypeSize(T) > 0 &&
+             " non positive amount of bits in __BitInt type");
+      assert(getContext().getTypeSize(T) <= 0xFFFFFFFF &&
+             " too many bits in __BitInt type");
+
+      // Redefine TypeKind with the actual __BitInt type if we have signed
+      // BitInt.
+      TypeKind = TK_BitInt;
+      IsBitInt = true;
+    }
   } else if (T->isFloatingType()) {
-    TypeKind = 1;
+    TypeKind = TK_Float;
     TypeInfo = getContext().getTypeSize(T);
   }
 
@@ -3314,6 +3349,20 @@ llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
       DiagnosticsEngine::ak_qualtype, (intptr_t)T.getAsOpaquePtr(), StringRef(),
       StringRef(), std::nullopt, Buffer, std::nullopt);
 
+  if (IsBitInt) {
+    // The Structure is: 0 to end the string, 32 bit unsigned integer in target
+    // endianness, zero.
+    char S[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
+    const auto *EIT = T->castAs<BitIntType>();
+    uint32_t Bits = EIT->getNumBits();
+    llvm::support::endian::write32(S + 1, Bits,
+                                   getTarget().isBigEndian()
+                                       ? llvm::endianness::big
+                                       : llvm::endianness::little);
+    StringRef Str = StringRef(S, sizeof(S) / sizeof(decltype(S[0])));
+    Buffer.append(Str);
+  }
+
   llvm::Constant *Components[] = {
     Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
     llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 6eac2b4c54e1ba3..84392745ea6144d 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -24,7 +24,6 @@
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
-#include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/CodeGenOptions.h"
@@ -196,24 +195,13 @@ static bool CanElideOverflowCheck(const ASTContext &Ctx, const BinOpInfo &Op) {
   if (!Op.mayHaveIntegerOverflow())
     return true;
 
-  const UnaryOperator *UO = dyn_cast<UnaryOperator>(Op.E);
-
-  if (UO && UO->getOpcode() == UO_Minus &&
-      Ctx.getLangOpts().isOverflowPatternExcluded(
-          LangOptions::OverflowPatternExclusionKind::NegUnsignedConst) &&
-      UO->isIntegerConstantExpr(Ctx))
-    return true;
-
   // If a unary op has a widened operand, the op cannot overflow.
-  if (UO)
+  if (const auto *UO = dyn_cast<UnaryOperator>(Op.E))
     return !UO->canOverflow();
 
   // We usually don't need overflow checks for binops with widened operands.
   // Multiplication with promoted unsigned operands is a special case.
   const auto *BO = cast<BinaryOperator>(Op.E);
-  if (BO->hasExcludedOverflowPattern())
-    return true;
-
   auto OptionalLHSTy = getUnwidenedIntegerType(Ctx, BO->getLHS());
   if (!OptionalLHSTy)
     return false;
@@ -2778,26 +2766,6 @@ llvm::Value *ScalarExprEmitter::EmitIncDecConsiderOverflowBehavior(
   llvm_unreachable("Unknown SignedOverflowBehaviorTy");
 }
 
-/// For the purposes of overflow pattern exclusion, does this match the
-/// "while(i--)" pattern?
-static bool matchesPostDecrInWhile(const UnaryOperator *UO, bool isInc,
-                                   bool isPre, ASTContext &Ctx) {
-  if (isInc || isPre)
-    return false;
-
-  // -fsanitize-overflow-pattern-exclusion=post-decr-while
-  if (!Ctx.getLangOpts().isOverflowPatternExcluded(
-          LangOptions::OverflowPatternExclusionKind::PostDecrInWhile))
-    return false;
-
-  // all Parents (usually just one) must be a WhileStmt
-  for (const auto &Parent : Ctx.getParentMapContext().getParents(*UO))
-    if (!Parent.get<WhileStmt>())
-      return false;
-
-  return true;
-}
-
 namespace {
 /// Handles check and update for lastprivate conditional variables.
 class OMPLastprivateConditionalUpdateRAII {
@@ -2909,10 +2877,6 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
   } else if (type->isIntegerType()) {
     QualType promotedType;
     bool canPerformLossyDemotionCheck = false;
-
-    bool excludeOverflowPattern =
-        matchesPostDecrInWhile(E, isInc, isPre, CGF.getContext());
-
     if (CGF.getContext().isPromotableIntegerType(type)) {
       promotedType = CGF.getContext().getPromotedIntegerType(type);
       assert(promotedType != type && "Shouldn't promote to the same type.");
@@ -2972,8 +2936,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     } else if (E->canOverflow() && type->isSignedIntegerOrEnumerationType()) {
       value = EmitIncDecConsiderOverflowBehavior(E, value, isInc);
     } else if (E->canOverflow() && type->isUnsignedIntegerType() &&
-               CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow) &&
-               !excludeOverflowPattern) {
+               CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) {
       value = EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(
           E, value, isInc, E->getFPFeaturesInEffect(CGF.getLangOpts())));
     } else {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 8c5e4aa9c037e2b..34120486996fb17 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -9588,15 +9588,18 @@ static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,
   genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);
 }
 
-static void emitNumTeamsForBareTargetDirective(
-    CodeGenFunction &CGF, const OMPExecutableDirective &D,
-    llvm::SmallVectorImpl<llvm::Value *> &NumTeams) {
-  const auto *C = D.getSingleClause<OMPNumTeamsClause>();
-  assert(!C->varlist_empty() && "ompx_bare requires explicit num_teams");
-  CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
-  for (auto *E : C->getNumTeams()) {
+template <typename ClauseTy>
+static void
+emitClauseForBareTargetDirective(CodeGenFunction &CGF,
+                                 const OMPExecutableDirective &D,
+                                 llvm::SmallVectorImpl<llvm::Value *> &Values) {
+  const auto *C = D.getSingleClause<ClauseTy>();
+  assert(!C->varlist_empty() &&
+         "ompx_bare requires explicit num_teams and thread_limit");
+  CodeGenFunction::RunCleanupsScope Scope(CGF);
+  for (auto *E : C->varlist()) {
     llvm::Value *V = CGF.EmitScalarExpr(E);
-    NumTeams.push_back(
+    Values.push_back(
         CGF.Builder.CreateIntCast(V, CGF.Int32Ty, /*isSigned=*/true));
   }
 }
@@ -9672,14 +9675,18 @@ static void emitTargetCallKernelLaunch(
 
     bool IsBare = D.hasClausesOfKind<OMPXBareClause>();
     SmallVector<llvm::Value *, 3> NumTeams;
-    if (IsBare)
-      emitNumTeamsForBareTargetDirective(CGF, D, NumTeams);
-    else
+    SmallVector<llvm::Value *, 3> NumThreads;
+    if (IsBare) {
+      emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, NumTeams);
+      emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,
+                                                             NumThreads);
+    } else {
       NumTeams.push_back(OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
+      NumThreads.push_back(
+          OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
+    }
 
     llvm::Value *DeviceID = emitDeviceID(Device, CGF);
-    llvm::Value *NumThreads =
-        OMPRuntime->emitNumThreadsForTargetDirective(CGF, D);
     llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc());
     llvm::Value *NumIterations =
         OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 6d11bd17d0a5f5d..0b61ef0f89989ce 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1220,8 +1220,14 @@ void CodeGenModule::Release() {
           (LangOpts.PointerAuthInitFiniAddressDiscrimination
            << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINIADDRDISC) |
           (LangOpts.PointerAuthELFGOT
-           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT);
-      static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT ==
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT) |
+          (LangOpts.PointerAuthIndirectGotos
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOTOS) |
+          (LangOpts.PointerAuthTypeInfoVTPtrDiscrimination
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_TYPEINFOVPTRDISCR) |
+          (LangOpts.PointerAuthFunctionTypeDiscrimination
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR);
+      static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR ==
                         AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST,
                     "Update when new enum items are defined");
       if (PAuthABIVersion != 0) {
diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp
index e4155810963eb81..989e46f4b66a7d6 100644
--- a/clang/lib/CodeGen/Targets/PPC.cpp
+++ b/clang/lib/CodeGen/Targets/PPC.cpp
@@ -217,8 +217,9 @@ ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
                                    /*Realign*/ TyAlign > CCAlign);
   }
 
-  return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
-                                     : ABIArgInfo::getDirect());
+  return (isPromotableTypeForABI(Ty)
+              ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
+              : ABIArgInfo::getDirect());
 }
 
 CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
@@ -891,8 +892,9 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
                                    /*Realign=*/TyAlign > ABIAlign);
   }
 
-  return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
-                                     : ABIArgInfo::getDirect());
+  return (isPromotableTypeForABI(Ty)
+              ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
+              : ABIArgInfo::getDirect());
 }
 
 ABIArgInfo
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 5b95019c25cab67..e12416e51f8d248 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -2271,7 +2271,8 @@ bool Driver::HandleImmediateArgs(Compilation &C) {
     return false;
   }
 
-  auto initializeTargets = [&]() {
+  if (C.getArgs().hasArg(options::OPT_print_libgcc_file_name)) {
+    ToolChain::RuntimeLibType RLT = TC.GetRuntimeLibType(C.getArgs());
     const llvm::Triple Triple(TC.ComputeEffectiveClangTriple(C.getArgs()));
     // The 'Darwin' toolchain is initialized only when its arguments are
     // computed. Get the default arguments for OFK_None to ensure that
@@ -2281,12 +2282,6 @@ bool Driver::HandleImmediateArgs(Compilation &C) {
     // FIXME: For some more esoteric targets the default toolchain is not the
     //        correct one.
     C.getArgsForToolChain(&TC, Triple.getArchName(), Action::OFK_None);
-    return Triple;
-  };
-
-  if (C.getArgs().hasArg(options::OPT_print_libgcc_file_name)) {
-    ToolChain::RuntimeLibType RLT = TC.GetRuntimeLibType(C.getArgs());
-    const llvm::Triple Triple = initializeTargets();
     RegisterEffectiveTriple TripleRAII(TC, Triple);
     switch (RLT) {
     case ToolChain::RLT_CompilerRT:
@@ -2330,9 +2325,7 @@ bool Driver::HandleImmediateArgs(Compilation &C) {
   }
 
   if (C.getArgs().hasArg(options::OPT_print_target_triple)) {
-    initializeTargets();
-    llvm::Triple Triple(TC.ComputeEffectiveClangTriple(C.getArgs()));
-    llvm::outs() << Triple.getTriple() << "\n";
+    llvm::outs() << TC.getTripleString() << "\n";
     return false;
   }
 
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index a63ee944fd1bb41..1fd870b72286e55 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -119,10 +119,6 @@ static SanitizerMask parseArgValues(const Driver &D, const llvm::opt::Arg *A,
 static int parseCoverageFeatures(const Driver &D, const llvm::opt::Arg *A,
                                  bool DiagnoseErrors);
 
-static int parseOverflowPatternExclusionValues(const Driver &D,
-                                               const llvm::opt::Arg *A,
-                                               bool DiagnoseErrors);
-
 /// Parse -f(no-)?sanitize-metadata= flag values, diagnosing any invalid
 /// components. Returns OR of members of \c BinaryMetadataFeature enumeration.
 static int parseBinaryMetadataFeatures(const Driver &D, const llvm::opt::Arg *A,
@@ -792,13 +788,6 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC,
           << "fsanitize-trap=cfi";
   }
 
-  for (const auto *Arg :
-       Args.filtered(options::OPT_fsanitize_overflow_pattern_exclusion_EQ)) {
-    Arg->claim();
-    OverflowPatternExclusions |=
-        parseOverflowPatternExclusionValues(D, Arg, DiagnoseErrors);
-  }
-
   // Parse -f(no-)?sanitize-coverage flags if coverage is supported by the
   // enabled sanitizers.
   for (const auto *Arg : Args) {
@@ -1252,10 +1241,6 @@ void SanitizerArgs::addArgs(const ToolChain &TC, const llvm::opt::ArgList &Args,
   addSpecialCaseListOpt(Args, CmdArgs,
                         "-fsanitize-system-ignorelist=", SystemIgnorelistFiles);
 
-  if (OverflowPatternExclusions)
-    Args.AddAllArgs(CmdArgs,
-                    options::OPT_fsanitize_overflow_pattern_exclusion_EQ);
-
   if (MsanTrackOrigins)
     CmdArgs.push_back(Args.MakeArgString("-fsanitize-memory-track-origins=" +
                                          Twine(MsanTrackOrigins)));
@@ -1441,28 +1426,6 @@ SanitizerMask parseArgValues(const Driver &D, const llvm::opt::Arg *A,
   return Kinds;
 }
 
-static int parseOverflowPatternExclusionValues(const Driver &D,
-                                               const llvm::opt::Arg *A,
-                                               bool DiagnoseErrors) {
-  int Exclusions = 0;
-  for (int i = 0, n = A->getNumValues(); i != n; ++i) {
-    const char *Value = A->getValue(i);
-    int E =
-        llvm::StringSwitch<int>(Value)
-            .Case("none", LangOptionsBase::None)
-            .Case("all", LangOptionsBase::All)
-            .Case("add-overflow-test", LangOptionsBase::AddOverflowTest)
-            .Case("negated-unsigned-const", LangOptionsBase::NegUnsignedConst)
-            .Case("post-decr-while", LangOptionsBase::PostDecrInWhile)
-            .Default(0);
-    if (E == 0)
-      D.Diag(clang::diag::err_drv_unsupported_option_argument)
-          << A->getSpelling() << Value;
-    Exclusions |= E;
-  }
-  return Exclusions;
-}
-
 int parseCoverageFeatures(const Driver &D, const llvm::opt::Arg *A,
                           bool DiagnoseErrors) {
   assert(A->getOption().matches(options::OPT_fsanitize_coverage) ||
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 013709999155217..0e8577b1115e38e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -9,6 +9,7 @@
 #include "clang/Driver/ToolChain.h"
 #include "ToolChains/Arch/AArch64.h"
 #include "ToolChains/Arch/ARM.h"
+#include "ToolChains/Arch/RISCV.h"
 #include "ToolChains/Clang.h"
 #include "ToolChains/CommonArgs.h"
 #include "ToolChains/Flang.h"
@@ -44,6 +45,7 @@
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/AArch64TargetParser.h"
+#include "llvm/TargetParser/RISCVISAInfo.h"
 #include "llvm/TargetParser/TargetParser.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cassert>
@@ -268,6 +270,19 @@ static void getARMMultilibFlags(const Driver &D,
   }
 }
 
+static void getRISCVMultilibFlags(const Driver &D, const llvm::Triple &Triple,
+                                  const llvm::opt::ArgList &Args,
+                                  Multilib::flags_list &Result) {
+  std::string Arch = riscv::getRISCVArch(Args, Triple);
+  // Canonicalize arch for easier matching
+  auto ISAInfo = llvm::RISCVISAInfo::parseArchString(
+      Arch, /*EnableExperimentalExtensions*/ true);
+  if (!llvm::errorToBool(ISAInfo.takeError()))
+    Result.push_back("-march=" + (*ISAInfo)->toString());
+
+  Result.push_back(("-mabi=" + riscv::getRISCVABI(Args, Triple)).str());
+}
+
 Multilib::flags_list
 ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   using namespace clang::driver::options;
@@ -288,6 +303,10 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   case llvm::Triple::thumbeb:
     getARMMultilibFlags(D, Triple, Args, Result);
     break;
+  case llvm::Triple::riscv32:
+  case llvm::Triple::riscv64:
+    getRISCVMultilibFlags(D, Triple, Args, Result);
+    break;
   default:
     break;
   }
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 149a31f58e75d21..6935904a24edbfd 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -95,69 +95,42 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple,
       CPUFastVectorUnaligned = true;
   }
 
-  // Handle features corresponding to "-ffixed-X" options
-  if (Args.hasArg(options::OPT_ffixed_x1))
-    Features.push_back("+reserve-x1");
-  if (Args.hasArg(options::OPT_ffixed_x2))
-    Features.push_back("+reserve-x2");
-  if (Args.hasArg(options::OPT_ffixed_x3))
-    Features.push_back("+reserve-x3");
-  if (Args.hasArg(options::OPT_ffixed_x4))
-    Features.push_back("+reserve-x4");
-  if (Args.hasArg(options::OPT_ffixed_x5))
-    Features.push_back("+reserve-x5");
-  if (Args.hasArg(options::OPT_ffixed_x6))
-    Features.push_back("+reserve-x6");
-  if (Args.hasArg(options::OPT_ffixed_x7))
-    Features.push_back("+reserve-x7");
-  if (Args.hasArg(options::OPT_ffixed_x8))
-    Features.push_back("+reserve-x8");
-  if (Args.hasArg(options::OPT_ffixed_x9))
-    Features.push_back("+reserve-x9");
-  if (Args.hasArg(options::OPT_ffixed_x10))
-    Features.push_back("+reserve-x10");
-  if (Args.hasArg(options::OPT_ffixed_x11))
-    Features.push_back("+reserve-x11");
-  if (Args.hasArg(options::OPT_ffixed_x12))
-    Features.push_back("+reserve-x12");
-  if (Args.hasArg(options::OPT_ffixed_x13))
-    Features.push_back("+reserve-x13");
-  if (Args.hasArg(options::OPT_ffixed_x14))
-    Features.push_back("+reserve-x14");
-  if (Args.hasArg(options::OPT_ffixed_x15))
-    Features.push_back("+reserve-x15");
-  if (Args.hasArg(options::OPT_ffixed_x16))
-    Features.push_back("+reserve-x16");
-  if (Args.hasArg(options::OPT_ffixed_x17))
-    Features.push_back("+reserve-x17");
-  if (Args.hasArg(options::OPT_ffixed_x18))
-    Features.push_back("+reserve-x18");
-  if (Args.hasArg(options::OPT_ffixed_x19))
-    Features.push_back("+reserve-x19");
-  if (Args.hasArg(options::OPT_ffixed_x20))
-    Features.push_back("+reserve-x20");
-  if (Args.hasArg(options::OPT_ffixed_x21))
-    Features.push_back("+reserve-x21");
-  if (Args.hasArg(options::OPT_ffixed_x22))
-    Features.push_back("+reserve-x22");
-  if (Args.hasArg(options::OPT_ffixed_x23))
-    Features.push_back("+reserve-x23");
-  if (Args.hasArg(options::OPT_ffixed_x24))
-    Features.push_back("+reserve-x24");
-  if (Args.hasArg(options::OPT_ffixed_x25))
-    Features.push_back("+reserve-x25");
-  if (Args.hasArg(options::OPT_ffixed_x26))
-    Features.push_back("+reserve-x26");
-  if (Args.hasArg(options::OPT_ffixed_x27))
-    Features.push_back("+reserve-x27");
-  if (Args.hasArg(options::OPT_ffixed_x28))
-    Features.push_back("+reserve-x28");
-  if (Args.hasArg(options::OPT_ffixed_x29))
-    Features.push_back("+reserve-x29");
-  if (Args.hasArg(options::OPT_ffixed_x30))
-    Features.push_back("+reserve-x30");
-  if (Args.hasArg(options::OPT_ffixed_x31))
-    Features.push_back("+reserve-x31");
+// Handle features corresponding to "-ffixed-X" options
+#define RESERVE_REG(REG)                                                       \
+  if (Args.hasArg(options::OPT_ffixed_##REG))                                  \
+    Features.push_back("+reserve-" #REG);
+  RESERVE_REG(x1)
+  RESERVE_REG(x2)
+  RESERVE_REG(x3)
+  RESERVE_REG(x4)
+  RESERVE_REG(x5)
+  RESERVE_REG(x6)
+  RESERVE_REG(x7)
+  RESERVE_REG(x8)
+  RESERVE_REG(x9)
+  RESERVE_REG(x10)
+  RESERVE_REG(x11)
+  RESERVE_REG(x12)
+  RESERVE_REG(x13)
+  RESERVE_REG(x14)
+  RESERVE_REG(x15)
+  RESERVE_REG(x16)
+  RESERVE_REG(x17)
+  RESERVE_REG(x18)
+  RESERVE_REG(x19)
+  RESERVE_REG(x20)
+  RESERVE_REG(x21)
+  RESERVE_REG(x22)
+  RESERVE_REG(x23)
+  RESERVE_REG(x24)
+  RESERVE_REG(x25)
+  RESERVE_REG(x26)
+  RESERVE_REG(x27)
+  RESERVE_REG(x28)
+  RESERVE_REG(x29)
+  RESERVE_REG(x30)
+  RESERVE_REG(x31)
+#undef RESERVE_REG
 
   // -mrelax is default, unless -mno-relax is specified.
   if (Args.hasFlag(options::OPT_mrelax, options::OPT_mno_relax, true)) {
diff --git a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
index ae1a4ba7882627f..5a1fedbec06adfa 100644
--- a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
@@ -179,6 +179,11 @@ void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args,
       Features.push_back("-hard-quad-float");
   }
 
+  if (Arg *A = Args.getLastArg(options::OPT_mv8plus, options::OPT_mno_v8plus)) {
+    if (A->getOption().matches(options::OPT_mv8plus))
+      Features.push_back("+v8plus");
+  }
+
   if (Args.hasArg(options::OPT_ffixed_g1))
     Features.push_back("+reserve-g1");
 
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 102d12700229e68..0f5f32f3f8f495a 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -218,17 +218,19 @@ static std::string computeBaseSysRoot(const Driver &D,
 void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
                               const ArgList &Args) {
   DetectedMultilibs Result;
-  if (isRISCVBareMetal(Triple)) {
+  // Look for a multilib.yaml before trying target-specific hardwired logic.
+  // If it exists, always do what it specifies.
+  llvm::SmallString<128> MultilibPath(computeBaseSysRoot(D, Triple));
+  llvm::sys::path::append(MultilibPath, MultilibFilename);
+  if (D.getVFS().exists(MultilibPath)) {
+    findMultilibsFromYAML(*this, D, MultilibPath, Args, Result);
+    SelectedMultilibs = Result.SelectedMultilibs;
+    Multilibs = Result.Multilibs;
+  } else if (isRISCVBareMetal(Triple)) {
     if (findRISCVMultilibs(D, Triple, Args, Result)) {
       SelectedMultilibs = Result.SelectedMultilibs;
       Multilibs = Result.Multilibs;
     }
-  } else {
-    llvm::SmallString<128> MultilibPath(computeBaseSysRoot(D, Triple));
-    llvm::sys::path::append(MultilibPath, MultilibFilename);
-    findMultilibsFromYAML(*this, D, MultilibPath, Args, Result);
-    SelectedMultilibs = Result.SelectedMultilibs;
-    Multilibs = Result.Multilibs;
   }
 }
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index f2bc11839edd4d0..a8606a866bdff64 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2104,6 +2104,9 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
     CmdArgs.push_back("-mabi=vec-extabi");
   }
 
+  if (!Args.hasFlag(options::OPT_mred_zone, options::OPT_mno_red_zone, true))
+    CmdArgs.push_back("-disable-red-zone");
+
   ppc::FloatABI FloatABI = ppc::getPPCFloatABI(D, Args);
   if (FloatABI == ppc::FloatABI::Soft) {
     // Floating point operations and argument passing are soft.
@@ -2123,42 +2126,6 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
   }
 }
 
-static void SetRISCVSmallDataLimit(const ToolChain &TC, const ArgList &Args,
-                                   ArgStringList &CmdArgs) {
-  const Driver &D = TC.getDriver();
-  const llvm::Triple &Triple = TC.getTriple();
-  // Default small data limitation is eight.
-  const char *SmallDataLimit = "8";
-  // Get small data limitation.
-  if (Args.getLastArg(options::OPT_shared, options::OPT_fpic,
-                      options::OPT_fPIC)) {
-    // Not support linker relaxation for PIC.
-    SmallDataLimit = "0";
-    if (Args.hasArg(options::OPT_G)) {
-      D.Diag(diag::warn_drv_unsupported_sdata);
-    }
-  } else if (Args.getLastArgValue(options::OPT_mcmodel_EQ)
-                 .equals_insensitive("large") &&
-             (Triple.getArch() == llvm::Triple::riscv64)) {
-    // Not support linker relaxation for RV64 with large code model.
-    SmallDataLimit = "0";
-    if (Args.hasArg(options::OPT_G)) {
-      D.Diag(diag::warn_drv_unsupported_sdata);
-    }
-  } else if (Triple.isAndroid()) {
-    // GP relaxation is not supported on Android.
-    SmallDataLimit = "0";
-    if (Args.hasArg(options::OPT_G)) {
-      D.Diag(diag::warn_drv_unsupported_sdata);
-    }
-  } else if (Arg *A = Args.getLastArg(options::OPT_G)) {
-    SmallDataLimit = A->getValue();
-  }
-  // Forward the -msmall-data-limit= option.
-  CmdArgs.push_back("-msmall-data-limit");
-  CmdArgs.push_back(SmallDataLimit);
-}
-
 void Clang::AddRISCVTargetArgs(const ArgList &Args,
                                ArgStringList &CmdArgs) const {
   const llvm::Triple &Triple = getToolChain().getTriple();
@@ -2167,7 +2134,10 @@ void Clang::AddRISCVTargetArgs(const ArgList &Args,
   CmdArgs.push_back("-target-abi");
   CmdArgs.push_back(ABIName.data());
 
-  SetRISCVSmallDataLimit(getToolChain(), Args, CmdArgs);
+  if (Arg *A = Args.getLastArg(options::OPT_G)) {
+    CmdArgs.push_back("-msmall-data-limit");
+    CmdArgs.push_back(A->getValue());
+  }
 
   if (!Args.hasFlag(options::OPT_mimplicit_float,
                     options::OPT_mno_implicit_float, true))
@@ -2582,6 +2552,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
   bool TakeNextArg = false;
 
   const llvm::Triple &Triple = C.getDefaultToolChain().getTriple();
+  bool IsELF = Triple.isOSBinFormatELF();
   bool Crel = false, ExperimentalCrel = false;
   bool UseRelaxRelocations = C.getDefaultToolChain().useRelaxRelocations();
   bool UseNoExecStack = false;
@@ -2620,11 +2591,29 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
           Value == "-mbig-obj")
         continue; // LLVM handles bigobj automatically
 
+      auto Equal = Value.split('=');
+      auto checkArg = [&](bool ValidTarget,
+                          std::initializer_list<const char *> Set) {
+        if (!ValidTarget) {
+          D.Diag(diag::err_drv_unsupported_opt_for_target)
+              << (Twine("-Wa,") + Equal.first + "=").str()
+              << Triple.getTriple();
+        } else if (!llvm::is_contained(Set, Equal.second)) {
+          D.Diag(diag::err_drv_unsupported_option_argument)
+              << (Twine("-Wa,") + Equal.first + "=").str() << Equal.second;
+        }
+      };
       switch (C.getDefaultToolChain().getArch()) {
       default:
         break;
       case llvm::Triple::x86:
       case llvm::Triple::x86_64:
+        if (Equal.first == "-mrelax-relocations" ||
+            Equal.first == "--mrelax-relocations") {
+          UseRelaxRelocations = Equal.second == "yes";
+          checkArg(IsELF, {"yes", "no"});
+          continue;
+        }
         if (Value == "-msse2avx") {
           CmdArgs.push_back("-msse2avx");
           continue;
@@ -2641,11 +2630,11 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
       case llvm::Triple::thumbeb:
       case llvm::Triple::arm:
       case llvm::Triple::armeb:
-        if (Value.starts_with("-mimplicit-it=")) {
+        if (Equal.first == "-mimplicit-it") {
           // Only store the value; the last value set takes effect.
-          ImplicitIt = Value.split("=").second;
-          if (CheckARMImplicitITArg(ImplicitIt))
-            continue;
+          ImplicitIt = Equal.second;
+          checkArg(true, {"always", "never", "arm", "thumb"});
+          continue;
         }
         if (Value == "-mthumb")
           // -mthumb has already been processed in ComputeLLVMTriple()
@@ -2719,12 +2708,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         Crel = false;
       } else if (Value == "--allow-experimental-crel") {
         ExperimentalCrel = true;
-      } else if (Value == "-mrelax-relocations=yes" ||
-                 Value == "--mrelax-relocations=yes") {
-        UseRelaxRelocations = true;
-      } else if (Value == "-mrelax-relocations=no" ||
-                 Value == "--mrelax-relocations=no") {
-        UseRelaxRelocations = false;
       } else if (Value.starts_with("-I")) {
         CmdArgs.push_back(Value.data());
         // We need to consume the next argument if the current arg is a plain
@@ -7769,9 +7752,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     Args.AddLastArg(CmdArgs, options::OPT_fgpu_default_stream_EQ);
   }
 
-  Args.AddAllArgs(CmdArgs,
-                  options::OPT_fsanitize_overflow_pattern_exclusion_EQ);
-
   Args.AddLastArg(CmdArgs, options::OPT_foffload_uniform_block,
                   options::OPT_fno_offload_uniform_block);
 
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 1cba3e1220264ab..320d2901da06ed5 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1143,8 +1143,8 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
   addMachineOutlinerArgs(D, Args, CmdArgs, ToolChain.getEffectiveTriple(),
                          /*IsLTO=*/true, PluginOptPrefix);
 
+  bool Crel = false;
   for (const Arg *A : Args.filtered(options::OPT_Wa_COMMA)) {
-    bool Crel = false;
     for (StringRef V : A->getValues()) {
       if (V == "--crel")
         Crel = true;
@@ -1154,13 +1154,13 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
         continue;
       A->claim();
     }
-    if (Crel) {
-      if (Triple.isOSBinFormatELF() && !Triple.isMIPS()) {
-        CmdArgs.push_back(Args.MakeArgString(Twine(PluginOptPrefix) + "-crel"));
-      } else {
-        D.Diag(diag::err_drv_unsupported_opt_for_target)
-            << "-Wa,--crel" << D.getTargetTriple();
-      }
+  }
+  if (Crel) {
+    if (Triple.isOSBinFormatELF() && !Triple.isMIPS()) {
+      CmdArgs.push_back(Args.MakeArgString(Twine(PluginOptPrefix) + "-crel"));
+    } else {
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << "-Wa,--crel" << D.getTargetTriple();
     }
   }
 }
diff --git a/clang/lib/Driver/ToolChains/HIPUtility.cpp b/clang/lib/Driver/ToolChains/HIPUtility.cpp
index f32a23f111e4bf9..1b707376dea819c 100644
--- a/clang/lib/Driver/ToolChains/HIPUtility.cpp
+++ b/clang/lib/Driver/ToolChains/HIPUtility.cpp
@@ -52,13 +52,16 @@ static std::string normalizeForBundler(const llvm::Triple &T,
 // input object or archive files.
 class HIPUndefinedFatBinSymbols {
 public:
-  HIPUndefinedFatBinSymbols(const Compilation &C)
-      : C(C), DiagID(C.getDriver().getDiags().getCustomDiagID(
-                  DiagnosticsEngine::Error,
-                  "Error collecting HIP undefined fatbin symbols: %0")),
+  HIPUndefinedFatBinSymbols(const Compilation &C,
+                            const llvm::opt::ArgList &Args_)
+      : C(C), Args(Args_),
+        DiagID(C.getDriver().getDiags().getCustomDiagID(
+            DiagnosticsEngine::Error,
+            "Error collecting HIP undefined fatbin symbols: %0")),
         Quiet(C.getArgs().hasArg(options::OPT__HASH_HASH_HASH)),
         Verbose(C.getArgs().hasArg(options::OPT_v)) {
     populateSymbols();
+    processStaticLibraries();
     if (Verbose) {
       for (const auto &Name : FatBinSymbols)
         llvm::errs() << "Found undefined HIP fatbin symbol: " << Name << "\n";
@@ -76,8 +79,70 @@ class HIPUndefinedFatBinSymbols {
     return GPUBinHandleSymbols;
   }
 
+  // Collect symbols from static libraries specified by -l options.
+  void processStaticLibraries() {
+    llvm::SmallVector<llvm::StringRef, 16> LibNames;
+    llvm::SmallVector<llvm::StringRef, 16> LibPaths;
+    llvm::SmallVector<llvm::StringRef, 16> ExactLibNames;
+    llvm::Triple Triple(C.getDriver().getTargetTriple());
+    bool IsMSVC = Triple.isWindowsMSVCEnvironment();
+    llvm::StringRef Ext = IsMSVC ? ".lib" : ".a";
+
+    for (const auto *Arg : Args.filtered(options::OPT_l)) {
+      llvm::StringRef Value = Arg->getValue();
+      if (Value.starts_with(":"))
+        ExactLibNames.push_back(Value.drop_front());
+      else
+        LibNames.push_back(Value);
+    }
+    for (const auto *Arg : Args.filtered(options::OPT_L)) {
+      auto Path = Arg->getValue();
+      LibPaths.push_back(Path);
+      if (Verbose)
+        llvm::errs() << "HIP fatbin symbol search uses library path:  " << Path
+                     << "\n";
+    }
+
+    auto ProcessLib = [&](llvm::StringRef LibName, bool IsExact) {
+      llvm::SmallString<256> FullLibName(
+          IsExact  ? Twine(LibName).str()
+          : IsMSVC ? (Twine(LibName) + Ext).str()
+                   : (Twine("lib") + LibName + Ext).str());
+
+      bool Found = false;
+      for (const auto Path : LibPaths) {
+        llvm::SmallString<256> FullPath = Path;
+        llvm::sys::path::append(FullPath, FullLibName);
+
+        if (llvm::sys::fs::exists(FullPath)) {
+          if (Verbose)
+            llvm::errs() << "HIP fatbin symbol search found library: "
+                         << FullPath << "\n";
+          auto BufferOrErr = llvm::MemoryBuffer::getFile(FullPath);
+          if (!BufferOrErr) {
+            errorHandler(llvm::errorCodeToError(BufferOrErr.getError()));
+            continue;
+          }
+          processInput(BufferOrErr.get()->getMemBufferRef());
+          Found = true;
+          break;
+        }
+      }
+      if (!Found && Verbose)
+        llvm::errs() << "HIP fatbin symbol search could not find library: "
+                     << FullLibName << "\n";
+    };
+
+    for (const auto LibName : ExactLibNames)
+      ProcessLib(LibName, true);
+
+    for (const auto LibName : LibNames)
+      ProcessLib(LibName, false);
+  }
+
 private:
   const Compilation &C;
+  const llvm::opt::ArgList &Args;
   unsigned DiagID;
   bool Quiet;
   bool Verbose;
@@ -301,7 +366,7 @@ void HIP::constructGenerateObjFileFromHIPFatBinary(
   auto HostTriple =
       C.getSingleOffloadToolChain<Action::OFK_Host>()->getTriple();
 
-  HIPUndefinedFatBinSymbols Symbols(C);
+  HIPUndefinedFatBinSymbols Symbols(C, Args);
 
   std::string PrimaryHipFatbinSymbol;
   std::string PrimaryGpuBinHandleSymbol;
diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index 8286e3be21803f2..ad44c2cfcd8115f 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -283,14 +283,6 @@ HLSLToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
     DAL->append(A);
   }
 
-  // Add default validator version if not set.
-  // TODO: remove this once read validator version from validator.
-  if (!DAL->hasArg(options::OPT_dxil_validator_version)) {
-    const StringRef DefaultValidatorVer = "1.7";
-    DAL->AddSeparateArg(nullptr,
-                        Opts.getOption(options::OPT_dxil_validator_version),
-                        DefaultValidatorVer);
-  }
   if (!DAL->hasArg(options::OPT_O_Group)) {
     DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_O), "3");
   }
diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 2b9b391c19c9fd0..3de45b00b4d00fd 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -170,6 +170,9 @@ bool types::isAcceptedByFlang(ID Id) {
   case TY_LLVM_IR:
   case TY_LLVM_BC:
     return true;
+  case TY_PP_CUDA:
+  case TY_CUDA:
+    return true;
   }
 }
 
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 6e56ee5b573f66a..1f8029cbd39ad25 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -104,6 +104,10 @@ Object serializePlatform(const Triple &T) {
   Object Platform;
   Platform["architecture"] = T.getArchName();
   Platform["vendor"] = T.getVendorName();
+
+  if (!T.getEnvironmentName().empty())
+    Platform["environment"] = T.getEnvironmentName();
+
   Platform["operatingSystem"] = serializeOperatingSystem(T);
   return Platform;
 }
@@ -171,22 +175,25 @@ std::optional<Array> serializeAvailability(const AvailabilityInfo &Avail) {
     UnconditionallyDeprecated["isUnconditionallyDeprecated"] = true;
     AvailabilityArray.emplace_back(std::move(UnconditionallyDeprecated));
   }
-  Object Availability;
-
-  Availability["domain"] = Avail.Domain;
-
-  if (Avail.isUnavailable()) {
-    Availability["isUnconditionallyUnavailable"] = true;
-  } else {
-    serializeObject(Availability, "introduced",
-                    serializeSemanticVersion(Avail.Introduced));
-    serializeObject(Availability, "deprecated",
-                    serializeSemanticVersion(Avail.Deprecated));
-    serializeObject(Availability, "obsoleted",
-                    serializeSemanticVersion(Avail.Obsoleted));
+
+  if (Avail.Domain.str() != "") {
+    Object Availability;
+    Availability["domain"] = Avail.Domain;
+
+    if (Avail.isUnavailable()) {
+      Availability["isUnconditionallyUnavailable"] = true;
+    } else {
+      serializeObject(Availability, "introduced",
+                      serializeSemanticVersion(Avail.Introduced));
+      serializeObject(Availability, "deprecated",
+                      serializeSemanticVersion(Avail.Deprecated));
+      serializeObject(Availability, "obsoleted",
+                      serializeSemanticVersion(Avail.Obsoleted));
+    }
+
+    AvailabilityArray.emplace_back(std::move(Availability));
   }
 
-  AvailabilityArray.emplace_back(std::move(Availability));
   return AvailabilityArray;
 }
 
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 43d246b7f824198..46dafad65863dcf 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1427,6 +1427,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
     switch (Style.RequiresClausePosition) {
     case FormatStyle::RCPS_OwnLine:
     case FormatStyle::RCPS_WithFollowing:
+    case FormatStyle::RCPS_OwnLineWithBrace:
       return CurrentState.Indent;
     default:
       break;
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 5358b35c19de25e..97fac41cdd30088 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -540,6 +540,7 @@ struct ScalarEnumerationTraits<FormatStyle::RequiresClausePositionStyle> {
   static void enumeration(IO &IO,
                           FormatStyle::RequiresClausePositionStyle &Value) {
     IO.enumCase(Value, "OwnLine", FormatStyle::RCPS_OwnLine);
+    IO.enumCase(Value, "OwnLineWithBrace", FormatStyle::RCPS_OwnLineWithBrace);
     IO.enumCase(Value, "WithPreceding", FormatStyle::RCPS_WithPreceding);
     IO.enumCase(Value, "WithFollowing", FormatStyle::RCPS_WithFollowing);
     IO.enumCase(Value, "SingleLine", FormatStyle::RCPS_SingleLine);
@@ -1904,7 +1905,6 @@ FormatStyle getGNUStyle() {
   Style.Cpp11BracedListStyle = false;
   Style.FixNamespaceComments = false;
   Style.SpaceBeforeParens = FormatStyle::SBPO_Always;
-  Style.Standard = FormatStyle::LS_Cpp03;
   return Style;
 }
 
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 9f79fa9fc516ca6..9d4204655b8ed6e 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -5681,6 +5681,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   if (Right.is(TT_RequiresClause)) {
     switch (Style.RequiresClausePosition) {
     case FormatStyle::RCPS_OwnLine:
+    case FormatStyle::RCPS_OwnLineWithBrace:
     case FormatStyle::RCPS_WithFollowing:
       return true;
     default:
@@ -5699,11 +5700,13 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
            (Style.BreakTemplateDeclarations == FormatStyle::BTDS_Leave &&
             Right.NewlinesBefore > 0);
   }
-  if (Left.ClosesRequiresClause && Right.isNot(tok::semi)) {
+  if (Left.ClosesRequiresClause) {
     switch (Style.RequiresClausePosition) {
     case FormatStyle::RCPS_OwnLine:
     case FormatStyle::RCPS_WithPreceding:
-      return true;
+      return Right.isNot(tok::semi);
+    case FormatStyle::RCPS_OwnLineWithBrace:
+      return !Right.isOneOf(tok::semi, tok::l_brace);
     default:
       break;
     }
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 5a5f5cb79a12f2b..e3911c281985b72 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -4267,19 +4267,6 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
       Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Val;
   }
 
-  if (auto *A = Args.getLastArg(OPT_fsanitize_overflow_pattern_exclusion_EQ)) {
-    for (int i = 0, n = A->getNumValues(); i != n; ++i) {
-      Opts.OverflowPatternExclusionMask |=
-          llvm::StringSwitch<unsigned>(A->getValue(i))
-              .Case("none", LangOptionsBase::None)
-              .Case("all", LangOptionsBase::All)
-              .Case("add-overflow-test", LangOptionsBase::AddOverflowTest)
-              .Case("negated-unsigned-const", LangOptionsBase::NegUnsignedConst)
-              .Case("post-decr-while", LangOptionsBase::PostDecrInWhile)
-              .Default(0);
-    }
-  }
-
   // Parse -fsanitize= arguments.
   parseSanitizerKinds("-fsanitize=", Args.getAllArgValues(OPT_fsanitize_EQ),
                       Diags, Opts.Sanitize);
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 8e62461d8a1818b..4f2856dd2247f8f 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -765,6 +765,7 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts,
   // C++26 features supported in earlier language modes.
   Builder.defineMacro("__cpp_pack_indexing", "202311L");
   Builder.defineMacro("__cpp_deleted_function", "202403L");
+  Builder.defineMacro("__cpp_variadic_friend", "202403L");
 
   if (LangOpts.Char8)
     Builder.defineMacro("__cpp_char8_t", "202207L");
diff --git a/clang/lib/Frontend/Rewrite/FixItRewriter.cpp b/clang/lib/Frontend/Rewrite/FixItRewriter.cpp
index 567bac576adb40a..44dfaf20eae73f0 100644
--- a/clang/lib/Frontend/Rewrite/FixItRewriter.cpp
+++ b/clang/lib/Frontend/Rewrite/FixItRewriter.cpp
@@ -21,8 +21,8 @@
 #include "clang/Edit/Commit.h"
 #include "clang/Edit/EditsReceiver.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
-#include "clang/Rewrite/Core/RewriteBuffer.h"
 #include "clang/Rewrite/Core/Rewriter.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/raw_ostream.h"
@@ -33,6 +33,7 @@
 #include <utility>
 
 using namespace clang;
+using llvm::RewriteBuffer;
 
 FixItRewriter::FixItRewriter(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
                              const LangOptions &LangOpts,
diff --git a/clang/lib/Frontend/Rewrite/HTMLPrint.cpp b/clang/lib/Frontend/Rewrite/HTMLPrint.cpp
index 69baa8f591088c9..5cce193ed1c4f30 100644
--- a/clang/lib/Frontend/Rewrite/HTMLPrint.cpp
+++ b/clang/lib/Frontend/Rewrite/HTMLPrint.cpp
@@ -20,8 +20,10 @@
 #include "clang/Rewrite/Core/HTMLRewrite.h"
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/Rewrite/Frontend/ASTConsumers.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
+using llvm::RewriteBuffer;
 
 //===----------------------------------------------------------------------===//
 // Functional HTML pretty-printing.
diff --git a/clang/lib/Frontend/Rewrite/RewriteMacros.cpp b/clang/lib/Frontend/Rewrite/RewriteMacros.cpp
index 5701b271aff103b..b533cabe97f5e9e 100644
--- a/clang/lib/Frontend/Rewrite/RewriteMacros.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteMacros.cpp
@@ -11,16 +11,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Frontend/Rewriters.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Rewrite/Frontend/Rewriters.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdio>
 #include <memory>
 
 using namespace clang;
+using llvm::RewriteBuffer;
 
 /// isSameToken - Return true if the two specified tokens start have the same
 /// content.
diff --git a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
index 3849e4040b53a17..f618c536b5f3c63 100644
--- a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Frontend/ASTConsumers.h"
 #include "clang/AST/AST.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/Attr.h"
@@ -23,6 +22,7 @@
 #include "clang/Config/config.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Rewrite/Frontend/ASTConsumers.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -34,6 +34,7 @@
 #if CLANG_ENABLE_OBJC_REWRITER
 
 using namespace clang;
+using llvm::RewriteBuffer;
 using llvm::utostr;
 
 namespace {
diff --git a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
index bf5176a2b6fb2e3..9db6ddbf0b8908f 100644
--- a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
@@ -32,6 +32,7 @@
 #if CLANG_ENABLE_OBJC_REWRITER
 
 using namespace clang;
+using llvm::RewriteBuffer;
 using llvm::utostr;
 
 namespace {
diff --git a/clang/lib/Headers/hlsl/hlsl_basic_types.h b/clang/lib/Headers/hlsl/hlsl_basic_types.h
index da6903df65ffed7..eff94e0d7f9500e 100644
--- a/clang/lib/Headers/hlsl/hlsl_basic_types.h
+++ b/clang/lib/Headers/hlsl/hlsl_basic_types.h
@@ -23,52 +23,98 @@ namespace hlsl {
 // 16-bit integer.
 typedef unsigned short uint16_t;
 typedef short int16_t;
+
+// 16-bit floating point.
+typedef half float16_t;
 #endif
 
+// 32-bit integer.
+typedef int int32_t;
+
 // unsigned 32-bit integer.
 typedef unsigned int uint;
+typedef unsigned int uint32_t;
+
+// 32-bit floating point.
+typedef float float32_t;
 
 // 64-bit integer.
 typedef unsigned long uint64_t;
 typedef long int64_t;
 
+// 64-bit floating point
+typedef double float64_t;
+
 // built-in vector data types:
 
 #ifdef __HLSL_ENABLE_16_BIT
+typedef vector<int16_t, 1> int16_t1;
 typedef vector<int16_t, 2> int16_t2;
 typedef vector<int16_t, 3> int16_t3;
 typedef vector<int16_t, 4> int16_t4;
+typedef vector<uint16_t, 1> uint16_t1;
 typedef vector<uint16_t, 2> uint16_t2;
 typedef vector<uint16_t, 3> uint16_t3;
 typedef vector<uint16_t, 4> uint16_t4;
 #endif
+typedef vector<bool, 1> bool1;
 typedef vector<bool, 2> bool2;
 typedef vector<bool, 3> bool3;
 typedef vector<bool, 4> bool4;
+typedef vector<int, 1> int1;
 typedef vector<int, 2> int2;
 typedef vector<int, 3> int3;
 typedef vector<int, 4> int4;
+typedef vector<uint, 1> uint1;
 typedef vector<uint, 2> uint2;
 typedef vector<uint, 3> uint3;
 typedef vector<uint, 4> uint4;
+typedef vector<int32_t, 1> int32_t1;
+typedef vector<int32_t, 2> int32_t2;
+typedef vector<int32_t, 3> int32_t3;
+typedef vector<int32_t, 4> int32_t4;
+typedef vector<uint32_t, 1> uint32_t1;
+typedef vector<uint32_t, 2> uint32_t2;
+typedef vector<uint32_t, 3> uint32_t3;
+typedef vector<uint32_t, 4> uint32_t4;
+typedef vector<int64_t, 1> int64_t1;
 typedef vector<int64_t, 2> int64_t2;
 typedef vector<int64_t, 3> int64_t3;
 typedef vector<int64_t, 4> int64_t4;
+typedef vector<uint64_t, 1> uint64_t1;
 typedef vector<uint64_t, 2> uint64_t2;
 typedef vector<uint64_t, 3> uint64_t3;
 typedef vector<uint64_t, 4> uint64_t4;
 
+typedef vector<half, 1> half1;
 typedef vector<half, 2> half2;
 typedef vector<half, 3> half3;
 typedef vector<half, 4> half4;
-
+typedef vector<float, 1> float1;
 typedef vector<float, 2> float2;
 typedef vector<float, 3> float3;
 typedef vector<float, 4> float4;
+typedef vector<double, 1> double1;
 typedef vector<double, 2> double2;
 typedef vector<double, 3> double3;
 typedef vector<double, 4> double4;
 
+#ifdef __HLSL_ENABLE_16_BIT
+typedef vector<float16_t, 1> float16_t1;
+typedef vector<float16_t, 2> float16_t2;
+typedef vector<float16_t, 3> float16_t3;
+typedef vector<float16_t, 4> float16_t4;
+#endif
+
+typedef vector<float32_t, 1> float32_t1;
+typedef vector<float32_t, 2> float32_t2;
+typedef vector<float32_t, 3> float32_t3;
+typedef vector<float32_t, 4> float32_t4;
+typedef vector<float64_t, 1> float64_t1;
+typedef vector<float64_t, 2> float64_t2;
+typedef vector<float64_t, 3> float64_t3;
+typedef vector<float64_t, 4> float64_t4;
+
 } // namespace hlsl
 
 #endif //_HLSL_HLSL_BASIC_TYPES_H_
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index aac89d910bbc839..18c5fe6056b4726 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -425,7 +425,10 @@ Decl *Parser::ParseLinkage(ParsingDeclSpec &DS, DeclaratorContext Context) {
       [[fallthrough]];
     default:
       ParsedAttributes DeclAttrs(AttrFactory);
-      MaybeParseCXX11Attributes(DeclAttrs);
+      ParsedAttributes DeclSpecAttrs(AttrFactory);
+      while (MaybeParseCXX11Attributes(DeclAttrs) ||
+             MaybeParseGNUAttributes(DeclSpecAttrs))
+        ;
       ParseExternalDeclaration(DeclAttrs, DeclSpecAttrs);
       continue;
     }
@@ -449,7 +452,7 @@ Decl *Parser::ParseLinkage(ParsingDeclSpec &DS, DeclaratorContext Context) {
 ///
 ///      export-function-declaration:
 ///         'export' function-declaration
-/// 
+///
 ///      export-declaration-group:
 ///         'export' '{' function-declaration-seq[opt] '}'
 ///
@@ -2004,9 +2007,16 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
 
   const PrintingPolicy &Policy = Actions.getASTContext().getPrintingPolicy();
   TagUseKind TUK;
-  if (isDefiningTypeSpecifierContext(DSC, getLangOpts().CPlusPlus) ==
-          AllowDefiningTypeSpec::No ||
-      (getLangOpts().OpenMP && OpenMPDirectiveParsing))
+
+  // C++26 [class.mem.general]p10: If a name-declaration matches the
+  // syntactic requirements of friend-type-declaration, it is a
+  // friend-type-declaration.
+  if (getLangOpts().CPlusPlus && DS.isFriendSpecifiedFirst() &&
+      Tok.isOneOf(tok::comma, tok::ellipsis))
+    TUK = TagUseKind::Friend;
+  else if (isDefiningTypeSpecifierContext(DSC, getLangOpts().CPlusPlus) ==
+               AllowDefiningTypeSpec::No ||
+           (getLangOpts().OpenMP && OpenMPDirectiveParsing))
     TUK = TagUseKind::Reference;
   else if (Tok.is(tok::l_brace) ||
            (DSC != DeclSpecContext::DSC_association &&
@@ -2238,9 +2248,28 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                             diag::err_keyword_not_allowed,
                             /*DiagnoseEmptyAttrs=*/true);
 
+    // Consume '...' first so we error on the ',' after it if there is one.
+    SourceLocation EllipsisLoc;
+    TryConsumeToken(tok::ellipsis, EllipsisLoc);
+
+    // CWG 2917: In a template-declaration whose declaration is a
+    // friend-type-declaration, the friend-type-specifier-list shall
+    // consist of exactly one friend-type-specifier.
+    //
+    // Essentially, the following is obviously nonsense, so disallow it:
+    //
+    //   template <typename>
+    //   friend class S, int;
+    //
+    if (Tok.is(tok::comma)) {
+      Diag(Tok.getLocation(),
+           diag::err_friend_template_decl_multiple_specifiers);
+      SkipUntil(tok::semi, StopBeforeMatch);
+    }
+
     TagOrTempResult = Actions.ActOnTemplatedFriendTag(
         getCurScope(), DS.getFriendSpecLoc(), TagType, StartLoc, SS, Name,
-        NameLoc, attrs,
+        NameLoc, EllipsisLoc, attrs,
         MultiTemplateParamsArg(TemplateParams ? &(*TemplateParams)[0] : nullptr,
                                TemplateParams ? TemplateParams->size() : 0));
   } else {
@@ -2815,6 +2844,7 @@ void Parser::MaybeParseAndDiagnoseDeclSpecAfterCXX11VirtSpecifierSeq(
 ///       member-declaration:
 ///         decl-specifier-seq[opt] member-declarator-list[opt] ';'
 ///         function-definition ';'[opt]
+/// [C++26] friend-type-declaration
 ///         ::[opt] nested-name-specifier template[opt] unqualified-id ';'[TODO]
 ///         using-declaration                                            [TODO]
 /// [C++0x] static_assert-declaration
@@ -2847,6 +2877,18 @@ void Parser::MaybeParseAndDiagnoseDeclSpecAfterCXX11VirtSpecifierSeq(
 ///       constant-initializer:
 ///         '=' constant-expression
 ///
+///       friend-type-declaration:
+///         'friend' friend-type-specifier-list ;
+///
+///       friend-type-specifier-list:
+///         friend-type-specifier ...[opt]
+///         friend-type-specifier-list , friend-type-specifier ...[opt]
+///
+///       friend-type-specifier:
+///         simple-type-specifier
+///         elaborated-type-specifier
+///         typename-specifier
+///
 Parser::DeclGroupPtrTy Parser::ParseCXXClassMemberDeclaration(
     AccessSpecifier AS, ParsedAttributes &AccessAttrs,
     ParsedTemplateInfo &TemplateInfo, ParsingDeclRAIIObject *TemplateDiags) {
@@ -3048,6 +3090,55 @@ Parser::DeclGroupPtrTy Parser::ParseCXXClassMemberDeclaration(
   if (DS.hasTagDefinition())
     Actions.ActOnDefinedDeclarationSpecifier(DS.getRepAsDecl());
 
+  // Handle C++26's variadic friend declarations. These don't even have
+  // declarators, so we get them out of the way early here.
+  if (DS.isFriendSpecifiedFirst() && Tok.isOneOf(tok::comma, tok::ellipsis)) {
+    Diag(Tok.getLocation(), getLangOpts().CPlusPlus26
+                                ? diag::warn_cxx23_variadic_friends
+                                : diag::ext_variadic_friends);
+
+    SourceLocation FriendLoc = DS.getFriendSpecLoc();
+    SmallVector<Decl *> Decls;
+
+    // Handles a single friend-type-specifier.
+    auto ParsedFriendDecl = [&](ParsingDeclSpec &DeclSpec) {
+      SourceLocation VariadicLoc;
+      TryConsumeToken(tok::ellipsis, VariadicLoc);
+
+      RecordDecl *AnonRecord = nullptr;
+      Decl *D = Actions.ParsedFreeStandingDeclSpec(
+          getCurScope(), AS, DeclSpec, DeclAttrs, TemplateParams, false,
+          AnonRecord, VariadicLoc);
+      DeclSpec.complete(D);
+      if (!D) {
+        SkipUntil(tok::semi, tok::r_brace);
+        return true;
+      }
+
+      Decls.push_back(D);
+      return false;
+    };
+
+    if (ParsedFriendDecl(DS))
+      return nullptr;
+
+    while (TryConsumeToken(tok::comma)) {
+      ParsingDeclSpec DeclSpec(*this, TemplateDiags);
+      const char *PrevSpec = nullptr;
+      unsigned DiagId = 0;
+      DeclSpec.SetFriendSpec(FriendLoc, PrevSpec, DiagId);
+      ParseDeclarationSpecifiers(DeclSpec, TemplateInfo, AS,
+                                 DeclSpecContext::DSC_class, nullptr);
+      if (ParsedFriendDecl(DeclSpec))
+        return nullptr;
+    }
+
+    ExpectAndConsume(tok::semi, diag::err_expected_semi_after_stmt,
+                     "friend declaration");
+
+    return Actions.BuildDeclaratorGroup(Decls);
+  }
+
   ParsingDeclarator DeclaratorInfo(*this, DS, DeclAttrs,
                                    DeclaratorContext::Member);
   if (TemplateInfo.TemplateParams)
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index b8e15b1aa6ce9e7..1405aef700bec5e 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -21,7 +21,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Availability.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/LocInfoType.h"
 #include "clang/Basic/PrettyStackTrace.h"
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Parse/Parser.h"
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 6a2088a73c55b91..dad39d8dc736de1 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/ExprObjC.h"
 #include "clang/AST/ODRDiagsEmitter.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
 #include "clang/Basic/CharInfo.h"
diff --git a/clang/lib/Rewrite/CMakeLists.txt b/clang/lib/Rewrite/CMakeLists.txt
index 16550b1b710ef9f..93ecb3c835260fe 100644
--- a/clang/lib/Rewrite/CMakeLists.txt
+++ b/clang/lib/Rewrite/CMakeLists.txt
@@ -3,9 +3,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_clang_library(clangRewrite
-  DeltaTree.cpp
   HTMLRewrite.cpp
-  RewriteRope.cpp
   Rewriter.cpp
   TokenRewriter.cpp
 
diff --git a/clang/lib/Rewrite/HTMLRewrite.cpp b/clang/lib/Rewrite/HTMLRewrite.cpp
index a96ca0764ae73d5..c75835df2c98ee8 100644
--- a/clang/lib/Rewrite/HTMLRewrite.cpp
+++ b/clang/lib/Rewrite/HTMLRewrite.cpp
@@ -16,6 +16,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/TokenConcatenation.h"
 #include "clang/Rewrite/Core/Rewriter.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
diff --git a/clang/lib/Rewrite/Rewriter.cpp b/clang/lib/Rewrite/Rewriter.cpp
index 0e6ae365064463b..68cf797f97905bc 100644
--- a/clang/lib/Rewrite/Rewriter.cpp
+++ b/clang/lib/Rewrite/Rewriter.cpp
@@ -17,8 +17,8 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Lexer.h"
-#include "clang/Rewrite/Core/RewriteBuffer.h"
-#include "clang/Rewrite/Core/RewriteRope.h"
+#include "llvm/ADT/RewriteBuffer.h"
+#include "llvm/ADT/RewriteRope.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
@@ -29,113 +29,18 @@
 #include <utility>
 
 using namespace clang;
+using llvm::RewriteBuffer;
 
-raw_ostream &RewriteBuffer::write(raw_ostream &os) const {
-  // Walk RewriteRope chunks efficiently using MoveToNextPiece() instead of the
-  // character iterator.
-  for (RopePieceBTreeIterator I = begin(), E = end(); I != E;
-       I.MoveToNextPiece())
-    os << I.piece();
-  return os;
-}
+//===----------------------------------------------------------------------===//
+// Rewriter class
+//===----------------------------------------------------------------------===//
 
 /// Return true if this character is non-new-line whitespace:
 /// ' ', '\\t', '\\f', '\\v', '\\r'.
 static inline bool isWhitespaceExceptNL(unsigned char c) {
-  switch (c) {
-  case ' ':
-  case '\t':
-  case '\f':
-  case '\v':
-  case '\r':
-    return true;
-  default:
-    return false;
-  }
+  return c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r';
 }
 
-void RewriteBuffer::RemoveText(unsigned OrigOffset, unsigned Size,
-                               bool removeLineIfEmpty) {
-  // Nothing to remove, exit early.
-  if (Size == 0) return;
-
-  unsigned RealOffset = getMappedOffset(OrigOffset, true);
-  assert(RealOffset+Size <= Buffer.size() && "Invalid location");
-
-  // Remove the dead characters.
-  Buffer.erase(RealOffset, Size);
-
-  // Add a delta so that future changes are offset correctly.
-  AddReplaceDelta(OrigOffset, -Size);
-
-  if (removeLineIfEmpty) {
-    // Find the line that the remove occurred and if it is completely empty
-    // remove the line as well.
-
-    iterator curLineStart = begin();
-    unsigned curLineStartOffs = 0;
-    iterator posI = begin();
-    for (unsigned i = 0; i != RealOffset; ++i) {
-      if (*posI == '\n') {
-        curLineStart = posI;
-        ++curLineStart;
-        curLineStartOffs = i + 1;
-      }
-      ++posI;
-    }
-
-    unsigned lineSize = 0;
-    posI = curLineStart;
-    while (posI != end() && isWhitespaceExceptNL(*posI)) {
-      ++posI;
-      ++lineSize;
-    }
-    if (posI != end() && *posI == '\n') {
-      Buffer.erase(curLineStartOffs, lineSize + 1/* + '\n'*/);
-      // FIXME: Here, the offset of the start of the line is supposed to be
-      // expressed in terms of the original input not the "real" rewrite
-      // buffer.  How do we compute that reliably?  It might be tempting to use
-      // curLineStartOffs + OrigOffset - RealOffset, but that assumes the
-      // difference between the original and real offset is the same at the
-      // removed text and at the start of the line, but that's not true if
-      // edits were previously made earlier on the line.  This bug is also
-      // documented by a FIXME on the definition of
-      // clang::Rewriter::RewriteOptions::RemoveLineIfEmpty.  A reproducer for
-      // the implementation below is the test RemoveLineIfEmpty in
-      // clang/unittests/Rewrite/RewriteBufferTest.cpp.
-      AddReplaceDelta(curLineStartOffs, -(lineSize + 1/* + '\n'*/));
-    }
-  }
-}
-
-void RewriteBuffer::InsertText(unsigned OrigOffset, StringRef Str,
-                               bool InsertAfter) {
-  // Nothing to insert, exit early.
-  if (Str.empty()) return;
-
-  unsigned RealOffset = getMappedOffset(OrigOffset, InsertAfter);
-  Buffer.insert(RealOffset, Str.begin(), Str.end());
-
-  // Add a delta so that future changes are offset correctly.
-  AddInsertDelta(OrigOffset, Str.size());
-}
-
-/// ReplaceText - This method replaces a range of characters in the input
-/// buffer with a new string.  This is effectively a combined "remove+insert"
-/// operation.
-void RewriteBuffer::ReplaceText(unsigned OrigOffset, unsigned OrigLength,
-                                StringRef NewStr) {
-  unsigned RealOffset = getMappedOffset(OrigOffset, true);
-  Buffer.erase(RealOffset, OrigLength);
-  Buffer.insert(RealOffset, NewStr.begin(), NewStr.end());
-  if (OrigLength != NewStr.size())
-    AddReplaceDelta(OrigOffset, NewStr.size() - OrigLength);
-}
-
-//===----------------------------------------------------------------------===//
-// Rewriter class
-//===----------------------------------------------------------------------===//
-
 /// getRangeSize - Return the size in bytes of the specified range if they
 /// are in the same file.  If not, this returns -1.
 int Rewriter::getRangeSize(const CharSourceRange &Range,
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 633b8220ffbf11d..29acd06af603be3 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
 #include "clang/AST/StmtCXX.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/Basic/DarwinSDKInfo.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/PartialDiagnostic.h"
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index 65b56bd1c8efc73..d0236d08c98e68d 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -433,6 +433,15 @@ static void ProcessAPINotes(Sema &S, VarDecl *D,
                   metadata);
 }
 
+/// Process API notes for a C field.
+static void ProcessAPINotes(Sema &S, FieldDecl *D,
+                            const api_notes::FieldInfo &Info,
+                            VersionedInfoMetadata metadata) {
+  // Handle common entity information.
+  ProcessAPINotes(S, D, static_cast<const api_notes::VariableInfo &>(Info),
+                  metadata);
+}
+
 /// Process API notes for an Objective-C property.
 static void ProcessAPINotes(Sema &S, ObjCPropertyDecl *D,
                             const api_notes::ObjCPropertyInfo &Info,
@@ -1062,6 +1071,17 @@ void Sema::ProcessAPINotes(Decl *D) {
       }
     }
 
+    if (auto Field = dyn_cast<FieldDecl>(D)) {
+      if (!Field->isUnnamedBitField() && !Field->isAnonymousStructOrUnion()) {
+        for (auto Reader : APINotes.findAPINotes(D->getLocation())) {
+          if (auto Context = UnwindTagContext(TagContext, APINotes)) {
+            auto Info = Reader->lookupField(Context->id, Field->getName());
+            ProcessVersionedAPINotes(*this, Field, Info);
+          }
+        }
+      }
+    }
+
     if (auto Tag = dyn_cast<TagDecl>(D)) {
       for (auto Reader : APINotes.findAPINotes(D->getLocation())) {
         if (auto Context = UnwindTagContext(TagContext, APINotes)) {
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index d8dd4fe16e3af00..e18872f0dc551e4 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -13,6 +13,7 @@
 #include "clang/Sema/SemaARM.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 641b180527da558..503e93f92571371 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -25,6 +25,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/NonTrivialTypeVisitor.h"
+#include "clang/AST/MangleNumberingContext.h"
 #include "clang/AST/Randstruct.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/Type.h"
@@ -5000,7 +5001,8 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
                                        const ParsedAttributesView &DeclAttrs,
                                        MultiTemplateParamsArg TemplateParams,
                                        bool IsExplicitInstantiation,
-                                       RecordDecl *&AnonRecord) {
+                                       RecordDecl *&AnonRecord,
+                                       SourceLocation EllipsisLoc) {
   Decl *TagD = nullptr;
   TagDecl *Tag = nullptr;
   if (DS.getTypeSpecType() == DeclSpec::TST_class ||
@@ -5067,9 +5069,12 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     // whatever routines created it handled the friendship aspect.
     if (TagD && !Tag)
       return nullptr;
-    return ActOnFriendTypeDecl(S, DS, TemplateParams);
+    return ActOnFriendTypeDecl(S, DS, TemplateParams, EllipsisLoc);
   }
 
+  assert(EllipsisLoc.isInvalid() &&
+         "Friend ellipsis but not friend-specified?");
+
   // Track whether this decl-specifier declares anything.
   bool DeclaresAnything = true;
 
@@ -6934,9 +6939,16 @@ static void checkAttributesAfterMerging(Sema &S, NamedDecl &ND) {
       // by applying it to the function type.
       if (const auto *A = ATL.getAttrAs<LifetimeBoundAttr>()) {
         const auto *MD = dyn_cast<CXXMethodDecl>(FD);
-        if (!MD || MD->isStatic()) {
+        int NoImplicitObjectError = -1;
+        if (!MD)
+          NoImplicitObjectError = 0;
+        else if (MD->isStatic())
+          NoImplicitObjectError = 1;
+        else if (MD->isExplicitObjectMemberFunction())
+          NoImplicitObjectError = 2;
+        if (NoImplicitObjectError != -1) {
           S.Diag(A->getLocation(), diag::err_lifetimebound_no_object_param)
-              << !MD << A->getRange();
+              << NoImplicitObjectError << A->getRange();
         } else if (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)) {
           S.Diag(A->getLocation(), diag::err_lifetimebound_ctor_dtor)
               << isa<CXXDestructorDecl>(MD) << A->getRange();
@@ -12226,12 +12238,9 @@ void Sema::CheckMain(FunctionDecl *FD, const DeclSpec &DS) {
   //    The main function shall not be declared with a linkage-specification.
   if (FD->isExternCContext() ||
       (FD->isExternCXXContext() &&
-       FD->getDeclContext()->getRedeclContext()->isTranslationUnit())) {
+       FD->getDeclContext()->getRedeclContext()->isTranslationUnit()))
     Diag(FD->getLocation(), diag::ext_main_invalid_linkage_specification)
         << FD->getLanguageLinkage();
-    FD->setInvalidDecl();
-    return;
-  }
 
   // C++11 [basic.start.main]p3:
   //   A program that [...] declares main to be inline, static or
@@ -18503,11 +18512,15 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
       // the program is ill-formed, except when compiling with MSVC extensions
       // enabled.
       if (EltTy->isReferenceType()) {
-        Diag(NewFD->getLocation(), getLangOpts().MicrosoftExt ?
-                                    diag::ext_union_member_of_reference_type :
-                                    diag::err_union_member_of_reference_type)
-          << NewFD->getDeclName() << EltTy;
-        if (!getLangOpts().MicrosoftExt)
+        const bool HaveMSExt =
+            getLangOpts().MicrosoftExt &&
+            !getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2015);
+
+        Diag(NewFD->getLocation(),
+             HaveMSExt ? diag::ext_union_member_of_reference_type
+                       : diag::err_union_member_of_reference_type)
+            << NewFD->getDeclName() << EltTy;
+        if (!HaveMSExt)
           NewFD->setInvalidDecl();
       }
     }
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 3b5e984f4ee7735..73d11ac972b020b 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7123,6 +7123,13 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   }
 }
 
+static bool isKernelDecl(Decl *D) {
+  const FunctionType *FnTy = D->getFunctionType();
+  return D->hasAttr<OpenCLKernelAttr>() ||
+         (FnTy && FnTy->getCallConv() == CallingConv::CC_AMDGPUKernelCall) ||
+         D->hasAttr<CUDAGlobalAttr>() || D->getAttr<NVPTXKernelAttr>();
+}
+
 void Sema::ProcessDeclAttributeList(
     Scope *S, Decl *D, const ParsedAttributesView &AttrList,
     const ProcessDeclAttributeOptions &Options) {
@@ -7163,24 +7170,25 @@ void Sema::ProcessDeclAttributeList(
     } else if (const auto *A = D->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
       Diag(D->getLocation(), diag::err_opencl_kernel_attr) << A;
       D->setInvalidDecl();
-    } else if (!D->hasAttr<CUDAGlobalAttr>()) {
-      if (const auto *A = D->getAttr<AMDGPUFlatWorkGroupSizeAttr>()) {
-        Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
-            << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
-        D->setInvalidDecl();
-      } else if (const auto *A = D->getAttr<AMDGPUWavesPerEUAttr>()) {
-        Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
-            << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
-        D->setInvalidDecl();
-      } else if (const auto *A = D->getAttr<AMDGPUNumSGPRAttr>()) {
-        Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
-            << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
-        D->setInvalidDecl();
-      } else if (const auto *A = D->getAttr<AMDGPUNumVGPRAttr>()) {
-        Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
-            << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
-        D->setInvalidDecl();
-      }
+    }
+  }
+  if (!isKernelDecl(D)) {
+    if (const auto *A = D->getAttr<AMDGPUFlatWorkGroupSizeAttr>()) {
+      Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
+          << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
+      D->setInvalidDecl();
+    } else if (const auto *A = D->getAttr<AMDGPUWavesPerEUAttr>()) {
+      Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
+          << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
+      D->setInvalidDecl();
+    } else if (const auto *A = D->getAttr<AMDGPUNumSGPRAttr>()) {
+      Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
+          << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
+      D->setInvalidDecl();
+    } else if (const auto *A = D->getAttr<AMDGPUNumVGPRAttr>()) {
+      Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
+          << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
+      D->setInvalidDecl();
     }
   }
 
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index c87edf41f35cee8..d89a47f3e6226a8 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -7056,11 +7056,16 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
         if (!RD->hasConstexprDestructor())
           return false;
 
+        QualType CanUnqualT = T.getCanonicalType().getUnqualifiedType();
         for (const CXXBaseSpecifier &B : RD->bases())
-          if (!Check(B.getType(), Check))
+          if (B.getType().getCanonicalType().getUnqualifiedType() !=
+                  CanUnqualT &&
+              !Check(B.getType(), Check))
             return false;
         for (const FieldDecl *FD : RD->fields())
-          if (!Check(FD->getType(), Check))
+          if (FD->getType().getCanonicalType().getUnqualifiedType() !=
+                  CanUnqualT &&
+              !Check(FD->getType(), Check))
             return false;
         return true;
       };
@@ -7644,9 +7649,13 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
     // parameter is of (possibly different) type “reference to C”,
     // in which case the type of F1 would differ from the type of F2
     // in that the type of F1 has an additional parameter;
-    if (!Context.hasSameType(
-            ThisType.getNonReferenceType().getUnqualifiedType(),
-            Context.getRecordType(RD))) {
+    QualType ExplicitObjectParameter = MD->isExplicitObjectMemberFunction()
+                                           ? MD->getParamDecl(0)->getType()
+                                           : QualType();
+    if (!ExplicitObjectParameter.isNull() &&
+        (!ExplicitObjectParameter->isReferenceType() ||
+         !Context.hasSameType(ExplicitObjectParameter.getNonReferenceType(),
+                              Context.getRecordType(RD)))) {
       if (DeleteOnTypeMismatch)
         ShouldDeleteForTypeMismatch = true;
       else {
@@ -8730,8 +8739,9 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
     // If we're out-of-class, this is the class we're comparing.
     if (!RD)
       RD = MD->getParent();
-    QualType T = MD->getFunctionObjectParameterType();
-    if (!T.isConstQualified()) {
+    QualType T = MD->getFunctionObjectParameterReferenceType();
+    if (!T.getNonReferenceType().isConstQualified() &&
+        (MD->isImplicitObjectMemberFunction() || T->isLValueReferenceType())) {
       SourceLocation Loc, InsertLoc;
       if (MD->isExplicitObjectMemberFunction()) {
         Loc = MD->getParamDecl(0)->getBeginLoc();
@@ -8750,11 +8760,17 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
       }
 
       // Add the 'const' to the type to recover.
-      const auto *FPT = MD->getType()->castAs<FunctionProtoType>();
-      FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
-      EPI.TypeQuals.addConst();
-      MD->setType(Context.getFunctionType(FPT->getReturnType(),
-                                          FPT->getParamTypes(), EPI));
+      if (MD->isExplicitObjectMemberFunction()) {
+        assert(T->isLValueReferenceType());
+        MD->getParamDecl(0)->setType(Context.getLValueReferenceType(
+            T.getNonReferenceType().withConst()));
+      } else {
+        const auto *FPT = MD->getType()->castAs<FunctionProtoType>();
+        FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
+        EPI.TypeQuals.addConst();
+        MD->setType(Context.getFunctionType(FPT->getReturnType(),
+                                            FPT->getParamTypes(), EPI));
+      }
     }
 
     if (MD->isVolatile()) {
@@ -8781,18 +8797,15 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
 
   const ParmVarDecl *KnownParm = nullptr;
   for (const ParmVarDecl *Param : FD->parameters()) {
-    if (Param->isExplicitObjectParameter())
-      continue;
     QualType ParmTy = Param->getType();
-
     if (!KnownParm) {
       auto CTy = ParmTy;
       // Is it `T const &`?
-      bool Ok = !IsMethod;
+      bool Ok = !IsMethod || FD->hasCXXExplicitFunctionObjectParameter();
       QualType ExpectedTy;
       if (RD)
         ExpectedTy = Context.getRecordType(RD);
-      if (auto *Ref = CTy->getAs<ReferenceType>()) {
+      if (auto *Ref = CTy->getAs<LValueReferenceType>()) {
         CTy = Ref->getPointeeType();
         if (RD)
           ExpectedTy.addConst();
@@ -8800,14 +8813,11 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
       }
 
       // Is T a class?
-      if (!Ok) {
-      } else if (RD) {
-        if (!RD->isDependentType() && !Context.hasSameType(CTy, ExpectedTy))
-          Ok = false;
-      } else if (auto *CRD = CTy->getAsRecordDecl()) {
-        RD = cast<CXXRecordDecl>(CRD);
+      if (RD) {
+        Ok &= RD->isDependentType() || Context.hasSameType(CTy, ExpectedTy);
       } else {
-        Ok = false;
+        RD = CTy->getAsCXXRecordDecl();
+        Ok &= RD != nullptr;
       }
 
       if (Ok) {
@@ -8847,7 +8857,7 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
     assert(FD->getFriendObjectKind() && "expected a friend declaration");
   } else {
     // Out of class, require the defaulted comparison to be a friend (of a
-    // complete type).
+    // complete type, per CWG2547).
     if (RequireCompleteType(FD->getLocation(), Context.getRecordType(RD),
                             diag::err_defaulted_comparison_not_friend, int(DCK),
                             int(1)))
@@ -17381,7 +17391,8 @@ Decl *Sema::BuildStaticAssertDeclaration(SourceLocation StaticAssertLoc,
 DeclResult Sema::ActOnTemplatedFriendTag(
     Scope *S, SourceLocation FriendLoc, unsigned TagSpec, SourceLocation TagLoc,
     CXXScopeSpec &SS, IdentifierInfo *Name, SourceLocation NameLoc,
-    const ParsedAttributesView &Attr, MultiTemplateParamsArg TempParamLists) {
+    SourceLocation EllipsisLoc, const ParsedAttributesView &Attr,
+    MultiTemplateParamsArg TempParamLists) {
   TagTypeKind Kind = TypeWithKeyword::getTagTypeKindForTypeSpec(TagSpec);
 
   bool IsMemberSpecialization = false;
@@ -17425,6 +17436,7 @@ DeclResult Sema::ActOnTemplatedFriendTag(
   // If it's explicit specializations all the way down, just forget
   // about the template header and build an appropriate non-templated
   // friend.  TODO: for source fidelity, remember the headers.
+  NestedNameSpecifierLoc QualifierLoc = SS.getWithLocInContext(Context);
   if (isAllExplicitSpecializations) {
     if (SS.isEmpty()) {
       bool Owned = false;
@@ -17440,7 +17452,6 @@ DeclResult Sema::ActOnTemplatedFriendTag(
                       /*IsTemplateParamOrArg=*/false, /*OOK=*/OOK_Outside);
     }
 
-    NestedNameSpecifierLoc QualifierLoc = SS.getWithLocInContext(Context);
     ElaboratedTypeKeyword Keyword
       = TypeWithKeyword::getKeywordForTagTypeKind(Kind);
     QualType T = CheckTypenameType(Keyword, TagLoc, QualifierLoc,
@@ -17462,8 +17473,9 @@ DeclResult Sema::ActOnTemplatedFriendTag(
       TL.getNamedTypeLoc().castAs<TypeSpecTypeLoc>().setNameLoc(NameLoc);
     }
 
-    FriendDecl *Friend = FriendDecl::Create(Context, CurContext, NameLoc,
-                                            TSI, FriendLoc, TempParamLists);
+    FriendDecl *Friend =
+        FriendDecl::Create(Context, CurContext, NameLoc, TSI, FriendLoc,
+                           EllipsisLoc, TempParamLists);
     Friend->setAccess(AS_public);
     CurContext->addDecl(Friend);
     return Friend;
@@ -17471,7 +17483,22 @@ DeclResult Sema::ActOnTemplatedFriendTag(
 
   assert(SS.isNotEmpty() && "valid templated tag with no SS and no direct?");
 
-
+  // CWG 2917: if it (= the friend-type-specifier) is a pack expansion
+  // (13.7.4 [temp.variadic]), any packs expanded by that pack expansion
+  // shall not have been introduced by the template-declaration.
+  SmallVector<UnexpandedParameterPack, 1> Unexpanded;
+  collectUnexpandedParameterPacks(QualifierLoc, Unexpanded);
+  unsigned FriendDeclDepth = TempParamLists.front()->getDepth();
+  for (UnexpandedParameterPack &U : Unexpanded) {
+    if (getDepthAndIndex(U).first >= FriendDeclDepth) {
+      auto *ND = U.first.dyn_cast<NamedDecl *>();
+      if (!ND)
+        ND = U.first.get<const TemplateTypeParmType *>()->getDecl();
+      Diag(U.second, diag::friend_template_decl_malformed_pack_expansion)
+          << ND->getDeclName() << SourceRange(SS.getBeginLoc(), EllipsisLoc);
+      return true;
+    }
+  }
 
   // Handle the case of a templated-scope friend class.  e.g.
   //   template <class T> class A<T>::B;
@@ -17486,8 +17513,9 @@ DeclResult Sema::ActOnTemplatedFriendTag(
   TL.setQualifierLoc(SS.getWithLocInContext(Context));
   TL.setNameLoc(NameLoc);
 
-  FriendDecl *Friend = FriendDecl::Create(Context, CurContext, NameLoc,
-                                          TSI, FriendLoc, TempParamLists);
+  FriendDecl *Friend =
+      FriendDecl::Create(Context, CurContext, NameLoc, TSI, FriendLoc,
+                         EllipsisLoc, TempParamLists);
   Friend->setAccess(AS_public);
   Friend->setUnsupportedFriend(true);
   CurContext->addDecl(Friend);
@@ -17495,7 +17523,8 @@ DeclResult Sema::ActOnTemplatedFriendTag(
 }
 
 Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
-                                MultiTemplateParamsArg TempParams) {
+                                MultiTemplateParamsArg TempParams,
+                                SourceLocation EllipsisLoc) {
   SourceLocation Loc = DS.getBeginLoc();
   SourceLocation FriendLoc = DS.getFriendSpecLoc();
 
@@ -17536,8 +17565,18 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   if (TheDeclarator.isInvalidType())
     return nullptr;
 
-  if (DiagnoseUnexpandedParameterPack(Loc, TSI, UPPC_FriendDeclaration))
+  // If '...' is present, the type must contain an unexpanded parameter
+  // pack, and vice versa.
+  bool Invalid = false;
+  if (EllipsisLoc.isInvalid() &&
+      DiagnoseUnexpandedParameterPack(Loc, TSI, UPPC_FriendDeclaration))
     return nullptr;
+  if (EllipsisLoc.isValid() &&
+      !TSI->getType()->containsUnexpandedParameterPack()) {
+    Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
+        << TSI->getTypeLoc().getSourceRange();
+    Invalid = true;
+  }
 
   if (!T->isElaboratedTypeSpecifier()) {
     if (TempParams.size()) {
@@ -17583,11 +17622,12 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
 
   Decl *D;
   if (!TempParams.empty())
+    // TODO: Support variadic friend template decls?
     D = FriendTemplateDecl::Create(Context, CurContext, Loc, TempParams, TSI,
                                    FriendLoc);
   else
     D = FriendDecl::Create(Context, CurContext, TSI->getTypeLoc().getBeginLoc(),
-                           TSI, FriendLoc);
+                           TSI, FriendLoc, EllipsisLoc);
 
   if (!D)
     return nullptr;
@@ -17595,6 +17635,9 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   D->setAccess(AS_public);
   CurContext->addDecl(D);
 
+  if (Invalid)
+    D->setInvalidDecl();
+
   return D;
 }
 
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index c4aa02ff0c217d2..f495658fe586416 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -25,6 +25,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExprOpenMP.h"
+#include "clang/AST/MangleNumberingContext.h"
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RecursiveASTVisitor.h"
@@ -6593,6 +6594,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
            "should only occur in error-recovery path.");
     return CallExpr::Create(Context, Fn, ArgExprs, Context.DependentTy,
                             VK_PRValue, RParenLoc, CurFPFeatureOverrides());
+  }
   return BuildResolvedCallExpr(Fn, NDecl, LParenLoc, ArgExprs, RParenLoc,
                                ExecConfig, IsExecConfig);
 }
diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp
index 2751c7cec2842e3..2f914ddc22a38b8 100644
--- a/clang/lib/Sema/SemaExprObjC.cpp
+++ b/clang/lib/Sema/SemaExprObjC.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Availability.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/StmtVisitor.h"
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 05d406a8630fbe4..a8461a7f6cf7123 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -14,6 +14,7 @@
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/MangleNumberingContext.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Initialization.h"
diff --git a/clang/lib/Sema/SemaMIPS.cpp b/clang/lib/Sema/SemaMIPS.cpp
index 269d927903c5d60..50d210e5b381405 100644
--- a/clang/lib/Sema/SemaMIPS.cpp
+++ b/clang/lib/Sema/SemaMIPS.cpp
@@ -13,6 +13,7 @@
 #include "clang/Sema/SemaMIPS.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Attr.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaObjC.cpp b/clang/lib/Sema/SemaObjC.cpp
index 75233689769c5bc..0359d18dd945251 100644
--- a/clang/lib/Sema/SemaObjC.cpp
+++ b/clang/lib/Sema/SemaObjC.cpp
@@ -17,6 +17,7 @@
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Attr.h"
+#include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 87d81dfaad601b4..74c646f64b42f20 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/OpenMPClause.h"
+#include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/StmtVisitor.h"
@@ -1255,16 +1256,8 @@ static const ValueDecl *getCanonicalDecl(const ValueDecl *D) {
   if (const auto *CED = dyn_cast<OMPCapturedExprDecl>(D))
     if (const auto *ME = dyn_cast<MemberExpr>(getExprAsWritten(CED->getInit())))
       D = ME->getMemberDecl();
-  const auto *VD = dyn_cast<VarDecl>(D);
-  const auto *FD = dyn_cast<FieldDecl>(D);
-  if (VD != nullptr) {
-    VD = VD->getCanonicalDecl();
-    D = VD;
-  } else {
-    assert(FD);
-    FD = FD->getCanonicalDecl();
-    D = FD;
-  }
+
+  D = cast<ValueDecl>(D->getCanonicalDecl());
   return D;
 }
 
@@ -3717,6 +3710,7 @@ namespace {
 class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
   DSAStackTy *Stack;
   Sema &SemaRef;
+  OpenMPDirectiveKind DKind = OMPD_unknown;
   bool ErrorFound = false;
   bool TryCaptureCXXThisMembers = false;
   CapturedStmt *CS = nullptr;
@@ -3748,7 +3742,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
     // Try to capture inner this->member references to generate correct mappings
     // and diagnostics.
     if (TryCaptureCXXThisMembers ||
-        (isOpenMPTargetExecutionDirective(Stack->getCurrentDirective()) &&
+        (isOpenMPTargetExecutionDirective(DKind) &&
          llvm::any_of(S->getInnermostCapturedStmt()->captures(),
                       [](const CapturedStmt::Capture &C) {
                         return C.capturesThis();
@@ -3818,7 +3812,6 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
         return;
 
       SourceLocation ELoc = E->getExprLoc();
-      OpenMPDirectiveKind DKind = Stack->getCurrentDirective();
       // The default(none) clause requires that each variable that is referenced
       // in the construct, and does not have a predetermined data-sharing
       // attribute, must have its data-sharing attribute explicitly determined
@@ -3983,7 +3976,6 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
         E->containsUnexpandedParameterPack() || E->isInstantiationDependent())
       return;
     auto *FD = dyn_cast<FieldDecl>(E->getMemberDecl());
-    OpenMPDirectiveKind DKind = Stack->getCurrentDirective();
     if (auto *TE = dyn_cast<CXXThisExpr>(E->getBase()->IgnoreParenCasts())) {
       if (!FD)
         return;
@@ -4022,7 +4014,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
         OpenMPDefaultmapClauseKind ClauseKind =
             getVariableCategoryFromDecl(SemaRef.getLangOpts(), FD);
         OpenMPMapClauseKind Kind = getMapClauseKindFromModifier(
-            Modifier, /*IsAggregateOrDeclareTarget*/ true);
+            Modifier, /*IsAggregateOrDeclareTarget=*/true);
         ImplicitMap[ClauseKind][Kind].emplace_back(E);
         return;
       }
@@ -4065,8 +4057,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
     if (isOpenMPTargetExecutionDirective(DKind)) {
       OMPClauseMappableExprCommon::MappableExprComponentList CurComponents;
       if (!checkMapClauseExpressionBase(SemaRef, E, CurComponents, OMPC_map,
-                                        Stack->getCurrentDirective(),
-                                        /*NoDiagnose=*/true))
+                                        DKind, /*NoDiagnose=*/true))
         return;
       const auto *VD = cast<ValueDecl>(
           CurComponents.back().getAssociatedDeclaration()->getCanonicalDecl());
@@ -4119,8 +4110,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
       // Skip analysis of arguments of implicitly defined map clause for target
       // directives.
       if (C && !((isa<OMPFirstprivateClause>(C) || isa<OMPMapClause>(C)) &&
-                 C->isImplicit() &&
-                 !isOpenMPTaskingDirective(Stack->getCurrentDirective()))) {
+                 C->isImplicit() && !isOpenMPTaskingDirective(DKind))) {
         for (Stmt *CC : C->children()) {
           if (CC)
             Visit(CC);
@@ -4169,7 +4159,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
       VarDecl *VD = Cap.getCapturedVar();
       // Do not try to map the variable if it or its sub-component was mapped
       // already.
-      if (isOpenMPTargetExecutionDirective(Stack->getCurrentDirective()) &&
+      if (isOpenMPTargetExecutionDirective(DKind) &&
           Stack->checkMappableExprComponentListsForDecl(
               VD, /*CurrentRegionOnly=*/true,
               [](OMPClauseMappableExprCommon::MappableExprComponentListRef,
@@ -4200,8 +4190,9 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
 
   DSAAttrChecker(DSAStackTy *S, Sema &SemaRef, CapturedStmt *CS)
       : Stack(S), SemaRef(SemaRef), ErrorFound(false), CS(CS) {
+    DKind = S->getCurrentDirective();
     // Process declare target link variables for the target directives.
-    if (isOpenMPTargetExecutionDirective(S->getCurrentDirective())) {
+    if (isOpenMPTargetExecutionDirective(DKind)) {
       for (DeclRefExpr *E : Stack->getLinkGlobals())
         Visit(E);
     }
@@ -4588,7 +4579,7 @@ static bool checkOrderedOrderSpecified(Sema &S,
 StmtResult SemaOpenMP::ActOnOpenMPRegionEnd(StmtResult S,
                                             ArrayRef<OMPClause *> Clauses) {
   handleDeclareVariantConstructTrait(DSAStack, DSAStack->getCurrentDirective(),
-                                     /* ScopeEntry */ false);
+                                     /*ScopeEntry=*/false);
   if (!isOpenMPCapturingDirective(DSAStack->getCurrentDirective()))
     return S;
 
@@ -7059,8 +7050,8 @@ void SemaOpenMP::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
     QualType UDeclTy = UDecl->getType();
     if (!UDeclTy->isDependentType()) {
       QualType NewType = getASTContext().mergeFunctionTypes(
-          FType, UDeclTy, /* OfBlockPointer */ false,
-          /* Unqualified */ false, /* AllowCXX */ true);
+          FType, UDeclTy, /*OfBlockPointer=*/false,
+          /*Unqualified=*/false, /*AllowCXX=*/true);
       if (NewType.isNull())
         continue;
     }
@@ -7106,9 +7097,8 @@ void SemaOpenMP::ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
     FD = cast<FunctionDecl>(D);
   auto *VariantFuncRef = DeclRefExpr::Create(
       getASTContext(), NestedNameSpecifierLoc(), SourceLocation(), FD,
-      /* RefersToEnclosingVariableOrCapture */ false,
-      /* NameLoc */ FD->getLocation(), FD->getType(),
-      ExprValueKind::VK_PRValue);
+      /*RefersToEnclosingVariableOrCapture=*/false,
+      /*NameLoc=*/FD->getLocation(), FD->getType(), ExprValueKind::VK_PRValue);
 
   OMPDeclareVariantScope &DVScope = OMPDeclareVariantScopes.back();
   auto *OMPDeclareVariantA = OMPDeclareVariantAttr::CreateImplicit(
@@ -7170,7 +7160,7 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
       OMPTraitInfo &TI = A->getTraitInfo();
       TI.getAsVariantMatchInfo(Context, VMI);
       if (!isVariantApplicableInContext(VMI, OMPCtx,
-                                        /* DeviceSetOnly */ false))
+                                        /*DeviceSetOnly=*/false))
         continue;
 
       VMIs.push_back(VMI);
@@ -7206,7 +7196,7 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
         auto *MemberCall = dyn_cast<CXXMemberCallExpr>(CE);
         BestExpr = MemberExpr::CreateImplicit(
             Context, MemberCall->getImplicitObjectArgument(),
-            /* IsArrow */ false, SpecializedMethod, Context.BoundMemberTy,
+            /*IsArrow=*/false, SpecializedMethod, Context.BoundMemberTy,
             MemberCall->getValueKind(), MemberCall->getObjectKind());
       }
       NewCall = SemaRef.BuildCallExpr(Scope, BestExpr, LParenLoc, ArgExprs,
@@ -7216,8 +7206,8 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
           FunctionDecl *NewCalleeFnDecl = NCE->getDirectCallee();
           QualType NewType = getASTContext().mergeFunctionTypes(
               CalleeFnType, NewCalleeFnDecl->getType(),
-              /* OfBlockPointer */ false,
-              /* Unqualified */ false, /* AllowCXX */ true);
+              /*OfBlockPointer=*/false,
+              /*Unqualified=*/false, /*AllowCXX=*/true);
           if (!NewType.isNull())
             break;
           // Don't use the call if the function type was not compatible.
@@ -7698,6 +7688,52 @@ struct LoopIterationSpace final {
   Expr *FinalCondition = nullptr;
 };
 
+/// Scan an AST subtree, checking that no decls in the CollapsedLoopVarDecls
+/// set are referenced.  Used for verifying loop nest structure before
+/// performing a loop collapse operation.
+class ForSubExprChecker final : public RecursiveASTVisitor<ForSubExprChecker> {
+  const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls;
+  VarDecl *ForbiddenVar = nullptr;
+  SourceRange ErrLoc;
+
+public:
+  explicit ForSubExprChecker(
+      const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls)
+      : CollapsedLoopVarDecls(CollapsedLoopVarDecls) {}
+
+  // We want to visit implicit code, i.e. synthetic initialisation statements
+  // created during range-for lowering.
+  bool shouldVisitImplicitCode() const { return true; }
+
+  bool VisitDeclRefExpr(DeclRefExpr *E) {
+    ValueDecl *VD = E->getDecl();
+    if (!isa<VarDecl, BindingDecl>(VD))
+      return true;
+    VarDecl *V = VD->getPotentiallyDecomposedVarDecl();
+    if (V->getType()->isReferenceType()) {
+      VarDecl *VD = V->getDefinition();
+      if (VD->hasInit()) {
+        Expr *I = VD->getInit();
+        DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(I);
+        if (!DRE)
+          return true;
+        V = DRE->getDecl()->getPotentiallyDecomposedVarDecl();
+      }
+    }
+    Decl *Canon = V->getCanonicalDecl();
+    if (CollapsedLoopVarDecls.contains(Canon)) {
+      ForbiddenVar = V;
+      ErrLoc = E->getSourceRange();
+      return false;
+    }
+
+    return true;
+  }
+
+  VarDecl *getForbiddenVar() const { return ForbiddenVar; }
+  SourceRange getErrRange() const { return ErrLoc; }
+};
+
 /// Helper class for checking canonical form of the OpenMP loops and
 /// extracting iteration space of each loop in the loop nest, that will be used
 /// for IR generation.
@@ -7712,6 +7748,8 @@ class OpenMPIterationSpaceChecker {
   SourceLocation DefaultLoc;
   /// A location for diagnostics (when increment is not compatible).
   SourceLocation ConditionLoc;
+  /// The set of variables declared within the (to be collapsed) loop nest.
+  const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls;
   /// A source location for referring to loop init later.
   SourceRange InitSrcRange;
   /// A source location for referring to condition later.
@@ -7755,10 +7793,13 @@ class OpenMPIterationSpaceChecker {
   Expr *Condition = nullptr;
 
 public:
-  OpenMPIterationSpaceChecker(Sema &SemaRef, bool SupportsNonRectangular,
-                              DSAStackTy &Stack, SourceLocation DefaultLoc)
+  OpenMPIterationSpaceChecker(
+      Sema &SemaRef, bool SupportsNonRectangular, DSAStackTy &Stack,
+      SourceLocation DefaultLoc,
+      const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopDecls)
       : SemaRef(SemaRef), SupportsNonRectangular(SupportsNonRectangular),
-        Stack(Stack), DefaultLoc(DefaultLoc), ConditionLoc(DefaultLoc) {}
+        Stack(Stack), DefaultLoc(DefaultLoc), ConditionLoc(DefaultLoc),
+        CollapsedLoopVarDecls(CollapsedLoopDecls) {}
   /// Check init-expr for canonical loop form and save loop counter
   /// variable - #Var and its initialization value - #LB.
   bool checkAndSetInit(Stmt *S, bool EmitDiags = true);
@@ -8079,6 +8120,16 @@ bool OpenMPIterationSpaceChecker::checkAndSetInit(Stmt *S, bool EmitDiags) {
     if (!ExprTemp->cleanupsHaveSideEffects())
       S = ExprTemp->getSubExpr();
 
+  if (!CollapsedLoopVarDecls.empty()) {
+    ForSubExprChecker FSEC{CollapsedLoopVarDecls};
+    if (!FSEC.TraverseStmt(S)) {
+      SourceRange Range = FSEC.getErrRange();
+      SemaRef.Diag(Range.getBegin(), diag::err_omp_loop_bad_collapse_var)
+          << Range.getEnd() << 0 << FSEC.getForbiddenVar();
+      return true;
+    }
+  }
+
   InitSrcRange = S->getSourceRange();
   if (Expr *E = dyn_cast<Expr>(S))
     S = E->IgnoreParens();
@@ -8182,6 +8233,17 @@ bool OpenMPIterationSpaceChecker::checkAndSetCond(Expr *S) {
   }
   Condition = S;
   S = getExprAsWritten(S);
+
+  if (!CollapsedLoopVarDecls.empty()) {
+    ForSubExprChecker FSEC{CollapsedLoopVarDecls};
+    if (!FSEC.TraverseStmt(S)) {
+      SourceRange Range = FSEC.getErrRange();
+      SemaRef.Diag(Range.getBegin(), diag::err_omp_loop_bad_collapse_var)
+          << Range.getEnd() << 1 << FSEC.getForbiddenVar();
+      return true;
+    }
+  }
+
   SourceLocation CondLoc = S->getBeginLoc();
   auto &&CheckAndSetCond =
       [this, IneqCondIsCanonical](BinaryOperatorKind Opcode, const Expr *LHS,
@@ -8280,6 +8342,16 @@ bool OpenMPIterationSpaceChecker::checkAndSetInc(Expr *S) {
     if (!ExprTemp->cleanupsHaveSideEffects())
       S = ExprTemp->getSubExpr();
 
+  if (!CollapsedLoopVarDecls.empty()) {
+    ForSubExprChecker FSEC{CollapsedLoopVarDecls};
+    if (!FSEC.TraverseStmt(S)) {
+      SourceRange Range = FSEC.getErrRange();
+      SemaRef.Diag(Range.getBegin(), diag::err_omp_loop_bad_collapse_var)
+          << Range.getEnd() << 2 << FSEC.getForbiddenVar();
+      return true;
+    }
+  }
+
   IncrementSrcRange = S->getSourceRange();
   S = S->IgnoreParens();
   if (auto *UO = dyn_cast<UnaryOperator>(S)) {
@@ -9001,8 +9073,9 @@ void SemaOpenMP::ActOnOpenMPLoopInitialization(SourceLocation ForLoc,
     return;
 
   DSAStack->loopStart();
+  llvm::SmallPtrSet<const Decl *, 1> EmptyDeclSet;
   OpenMPIterationSpaceChecker ISC(SemaRef, /*SupportsNonRectangular=*/true,
-                                  *DSAStack, ForLoc);
+                                  *DSAStack, ForLoc, EmptyDeclSet);
   if (!ISC.checkAndSetInit(Init, /*EmitDiags=*/false)) {
     if (ValueDecl *D = ISC.getLoopDecl()) {
       auto *VD = dyn_cast<VarDecl>(D);
@@ -9099,7 +9172,8 @@ static bool checkOpenMPIterationSpace(
     Expr *OrderedLoopCountExpr,
     SemaOpenMP::VarsWithInheritedDSAType &VarsWithImplicitDSA,
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
-    llvm::MapVector<const Expr *, DeclRefExpr *> &Captures) {
+    llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
+    const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
   // OpenMP [2.9.1, Canonical Loop Form]
   //   for (init-expr; test-expr; incr-expr) structured-block
@@ -9138,7 +9212,8 @@ static bool checkOpenMPIterationSpace(
     return false;
 
   OpenMPIterationSpaceChecker ISC(SemaRef, SupportsNonRectangular, DSA,
-                                  For ? For->getForLoc() : CXXFor->getForLoc());
+                                  For ? For->getForLoc() : CXXFor->getForLoc(),
+                                  CollapsedLoopVarDecls);
 
   // Check init.
   Stmt *Init = For ? For->getInit() : CXXFor->getBeginStmt();
@@ -9415,7 +9490,7 @@ static ExprResult widenIterationCount(unsigned Bits, Expr *E, Sema &SemaRef) {
   if (HasBits >= Bits)
     return ExprResult(E);
   // OK to convert to signed, because new type has more bits than old.
-  QualType NewType = C.getIntTypeForBitwidth(Bits, /* Signed */ true);
+  QualType NewType = C.getIntTypeForBitwidth(Bits, /*Signed=*/true);
   return SemaRef.PerformImplicitConversion(E, NewType, Sema::AA_Converting,
                                            true);
 }
@@ -9505,6 +9580,39 @@ static Expr *buildPostUpdate(Sema &S, ArrayRef<Expr *> PostUpdates) {
   return PostUpdate;
 }
 
+/// Look for variables declared in the body parts of a for-loop nest.  Used
+/// for verifying loop nest structure before performing a loop collapse
+/// operation.
+class ForVarDeclFinder final : public RecursiveASTVisitor<ForVarDeclFinder> {
+  int NestingDepth = 0;
+  llvm::SmallPtrSetImpl<const Decl *> &VarDecls;
+
+public:
+  explicit ForVarDeclFinder(llvm::SmallPtrSetImpl<const Decl *> &VD)
+      : VarDecls(VD) {}
+
+  bool VisitForStmt(ForStmt *F) {
+    ++NestingDepth;
+    TraverseStmt(F->getBody());
+    --NestingDepth;
+    return false;
+  }
+
+  bool VisitCXXForRangeStmt(CXXForRangeStmt *RF) {
+    ++NestingDepth;
+    TraverseStmt(RF->getBody());
+    --NestingDepth;
+    return false;
+  }
+
+  bool VisitVarDecl(VarDecl *D) {
+    Decl *C = D->getCanonicalDecl();
+    if (NestingDepth > 0)
+      VarDecls.insert(C);
+    return true;
+  }
+};
+
 /// Called on a for stmt to check itself and nested loops (if any).
 /// \return Returns 0 if one of the collapsed stmts is not canonical for loop,
 /// number of collapsed loops otherwise.
@@ -9517,6 +9625,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
   unsigned NestedLoopCount = 1;
   bool SupportsNonPerfectlyNested = (SemaRef.LangOpts.OpenMP >= 50) &&
                                     !isOpenMPLoopTransformationDirective(DKind);
+  llvm::SmallPtrSet<const Decl *, 4> CollapsedLoopVarDecls;
 
   if (CollapseLoopCountExpr) {
     // Found 'collapse' clause - calculate collapse number.
@@ -9524,6 +9633,9 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     if (!CollapseLoopCountExpr->isValueDependent() &&
         CollapseLoopCountExpr->EvaluateAsInt(Result, SemaRef.getASTContext())) {
       NestedLoopCount = Result.Val.getInt().getLimitedValue();
+
+      ForVarDeclFinder FVDF{CollapsedLoopVarDecls};
+      FVDF.TraverseStmt(AStmt);
     } else {
       Built.clear(/*Size=*/1);
       return 1;
@@ -9561,11 +9673,13 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
           SupportsNonPerfectlyNested, NumLoops,
           [DKind, &SemaRef, &DSA, NumLoops, NestedLoopCount,
            CollapseLoopCountExpr, OrderedLoopCountExpr, &VarsWithImplicitDSA,
-           &IterSpaces, &Captures](unsigned Cnt, Stmt *CurStmt) {
+           &IterSpaces, &Captures,
+           &CollapsedLoopVarDecls](unsigned Cnt, Stmt *CurStmt) {
             if (checkOpenMPIterationSpace(
                     DKind, CurStmt, SemaRef, DSA, Cnt, NestedLoopCount,
                     NumLoops, CollapseLoopCountExpr, OrderedLoopCountExpr,
-                    VarsWithImplicitDSA, IterSpaces, Captures))
+                    VarsWithImplicitDSA, IterSpaces, Captures,
+                    CollapsedLoopVarDecls))
               return true;
             if (Cnt > 0 && Cnt >= NestedLoopCount &&
                 IterSpaces[Cnt].CounterVar) {
@@ -9600,7 +9714,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
           }))
     return 0;
 
-  Built.clear(/* size */ NestedLoopCount);
+  Built.clear(/*size=*/NestedLoopCount);
 
   if (SemaRef.CurContext->isDependentContext())
     return NestedLoopCount;
@@ -9754,13 +9868,13 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     LB = buildDeclRefExpr(SemaRef, LBDecl, VType, InitLoc);
     SemaRef.AddInitializerToDecl(LBDecl,
                                  SemaRef.ActOnIntegerConstant(InitLoc, 0).get(),
-                                 /*DirectInit*/ false);
+                                 /*DirectInit=*/false);
 
     // Upper bound variable, initialized with last iteration number.
     VarDecl *UBDecl = buildVarDecl(SemaRef, InitLoc, VType, ".omp.ub");
     UB = buildDeclRefExpr(SemaRef, UBDecl, VType, InitLoc);
     SemaRef.AddInitializerToDecl(UBDecl, LastIteration.get(),
-                                 /*DirectInit*/ false);
+                                 /*DirectInit=*/false);
 
     // A 32-bit variable-flag where runtime returns 1 for the last iteration.
     // This will be used to implement clause 'lastprivate'.
@@ -9769,7 +9883,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     IL = buildDeclRefExpr(SemaRef, ILDecl, Int32Ty, InitLoc);
     SemaRef.AddInitializerToDecl(ILDecl,
                                  SemaRef.ActOnIntegerConstant(InitLoc, 0).get(),
-                                 /*DirectInit*/ false);
+                                 /*DirectInit=*/false);
 
     // Stride variable returned by runtime (we initialize it to 1 by default).
     VarDecl *STDecl =
@@ -9777,7 +9891,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     ST = buildDeclRefExpr(SemaRef, STDecl, StrideVType, InitLoc);
     SemaRef.AddInitializerToDecl(STDecl,
                                  SemaRef.ActOnIntegerConstant(InitLoc, 1).get(),
-                                 /*DirectInit*/ false);
+                                 /*DirectInit=*/false);
 
     // Build expression: UB = min(UB, LastIteration)
     // It is necessary for CodeGen of directives with static scheduling.
@@ -9788,7 +9902,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
         LastIteration.get(), UB.get());
     EUB = SemaRef.BuildBinOp(CurScope, InitLoc, BO_Assign, UB.get(),
                              CondOp.get());
-    EUB = SemaRef.ActOnFinishFullExpr(EUB.get(), /*DiscardedValue*/ false);
+    EUB = SemaRef.ActOnFinishFullExpr(EUB.get(), /*DiscardedValue=*/false);
 
     // If we have a combined directive that combines 'distribute', 'for' or
     // 'simd' we need to be able to access the bounds of the schedule of the
@@ -9801,14 +9915,14 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombLB = buildDeclRefExpr(SemaRef, CombLBDecl, VType, InitLoc);
       SemaRef.AddInitializerToDecl(
           CombLBDecl, SemaRef.ActOnIntegerConstant(InitLoc, 0).get(),
-          /*DirectInit*/ false);
+          /*DirectInit=*/false);
 
       // Upper bound variable, initialized with last iteration number.
       VarDecl *CombUBDecl =
           buildVarDecl(SemaRef, InitLoc, VType, ".omp.comb.ub");
       CombUB = buildDeclRefExpr(SemaRef, CombUBDecl, VType, InitLoc);
       SemaRef.AddInitializerToDecl(CombUBDecl, LastIteration.get(),
-                                   /*DirectInit*/ false);
+                                   /*DirectInit=*/false);
 
       ExprResult CombIsUBGreater = SemaRef.BuildBinOp(
           CurScope, InitLoc, BO_GT, CombUB.get(), LastIteration.get());
@@ -9818,7 +9932,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombEUB = SemaRef.BuildBinOp(CurScope, InitLoc, BO_Assign, CombUB.get(),
                                    CombCondOp.get());
       CombEUB =
-          SemaRef.ActOnFinishFullExpr(CombEUB.get(), /*DiscardedValue*/ false);
+          SemaRef.ActOnFinishFullExpr(CombEUB.get(), /*DiscardedValue=*/false);
 
       const CapturedDecl *CD = cast<CapturedStmt>(AStmt)->getCapturedDecl();
       // We expect to have at least 2 more parameters than the 'parallel'
@@ -9854,7 +9968,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                     ? LB.get()
                     : SemaRef.ActOnIntegerConstant(SourceLocation(), 0).get();
     Init = SemaRef.BuildBinOp(CurScope, InitLoc, BO_Assign, IV.get(), RHS);
-    Init = SemaRef.ActOnFinishFullExpr(Init.get(), /*DiscardedValue*/ false);
+    Init = SemaRef.ActOnFinishFullExpr(Init.get(), /*DiscardedValue=*/false);
 
     if (isOpenMPLoopBoundSharingDirective(DKind)) {
       Expr *CombRHS =
@@ -9867,7 +9981,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombInit =
           SemaRef.BuildBinOp(CurScope, InitLoc, BO_Assign, IV.get(), CombRHS);
       CombInit =
-          SemaRef.ActOnFinishFullExpr(CombInit.get(), /*DiscardedValue*/ false);
+          SemaRef.ActOnFinishFullExpr(CombInit.get(), /*DiscardedValue=*/false);
     }
   }
 
@@ -9887,7 +10001,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                         SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get())
             .get();
     BoundUB =
-        SemaRef.ActOnFinishFullExpr(BoundUB, /*DiscardedValue*/ false).get();
+        SemaRef.ActOnFinishFullExpr(BoundUB, /*DiscardedValue=*/false).get();
   }
   ExprResult Cond =
       (isOpenMPWorksharingDirective(DKind) ||
@@ -9916,7 +10030,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                   SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get())
               .get();
       BoundCombUB =
-          SemaRef.ActOnFinishFullExpr(BoundCombUB, /*DiscardedValue*/ false)
+          SemaRef.ActOnFinishFullExpr(BoundCombUB, /*DiscardedValue=*/false)
               .get();
     }
     CombCond =
@@ -9931,7 +10045,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
   if (!Inc.isUsable())
     return 0;
   Inc = SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, IV.get(), Inc.get());
-  Inc = SemaRef.ActOnFinishFullExpr(Inc.get(), /*DiscardedValue*/ false);
+  Inc = SemaRef.ActOnFinishFullExpr(Inc.get(), /*DiscardedValue=*/false);
   if (!Inc.isUsable())
     return 0;
 
@@ -9952,7 +10066,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     NextLB =
         SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, LB.get(), NextLB.get());
     NextLB =
-        SemaRef.ActOnFinishFullExpr(NextLB.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(NextLB.get(), /*DiscardedValue=*/false);
     if (!NextLB.isUsable())
       return 0;
     // UB + ST
@@ -9963,7 +10077,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     NextUB =
         SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, UB.get(), NextUB.get());
     NextUB =
-        SemaRef.ActOnFinishFullExpr(NextUB.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(NextUB.get(), /*DiscardedValue=*/false);
     if (!NextUB.isUsable())
       return 0;
     if (isOpenMPLoopBoundSharingDirective(DKind)) {
@@ -9975,7 +10089,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombNextLB = SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, CombLB.get(),
                                       CombNextLB.get());
       CombNextLB = SemaRef.ActOnFinishFullExpr(CombNextLB.get(),
-                                               /*DiscardedValue*/ false);
+                                               /*DiscardedValue=*/false);
       if (!CombNextLB.isUsable())
         return 0;
       // UB + ST
@@ -9987,7 +10101,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombNextUB = SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, CombUB.get(),
                                       CombNextUB.get());
       CombNextUB = SemaRef.ActOnFinishFullExpr(CombNextUB.get(),
-                                               /*DiscardedValue*/ false);
+                                               /*DiscardedValue=*/false);
       if (!CombNextUB.isUsable())
         return 0;
     }
@@ -10010,7 +10124,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     DistInc = SemaRef.BuildBinOp(CurScope, DistIncLoc, BO_Assign, IV.get(),
                                  DistInc.get());
     DistInc =
-        SemaRef.ActOnFinishFullExpr(DistInc.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(DistInc.get(), /*DiscardedValue=*/false);
     assert(DistInc.isUsable() && "distribute inc expr was not built");
 
     // Build expression: UB = min(UB, prevUB) for #for in composite or combined
@@ -10033,7 +10147,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     PrevEUB = SemaRef.BuildBinOp(CurScope, DistIncLoc, BO_Assign, UB.get(),
                                  CondOp.get());
     PrevEUB =
-        SemaRef.ActOnFinishFullExpr(PrevEUB.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(PrevEUB.get(), /*DiscardedValue=*/false);
 
     // Build IV <= PrevUB or IV < PrevUB + 1 for unsigned IV to be used in
     // parallel for is in combination with a distribute directive with
@@ -10047,7 +10161,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                   SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get())
               .get();
       BoundPrevUB =
-          SemaRef.ActOnFinishFullExpr(BoundPrevUB, /*DiscardedValue*/ false)
+          SemaRef.ActOnFinishFullExpr(BoundPrevUB, /*DiscardedValue=*/false)
               .get();
     }
     ParForInDistCond =
@@ -15319,7 +15433,7 @@ SemaOpenMP::PerformOpenMPImplicitIntegerConversion(SourceLocation Loc,
   class IntConvertDiagnoser : public Sema::ICEConvertDiagnoser {
   public:
     IntConvertDiagnoser()
-        : ICEConvertDiagnoser(/*AllowScopedEnumerations*/ false, false, true) {}
+        : ICEConvertDiagnoser(/*AllowScopedEnumerations=*/false, false, true) {}
     SemaDiagnosticBuilder diagnoseNotInt(Sema &S, SourceLocation Loc,
                                          QualType T) override {
       return S.Diag(Loc, diag::err_omp_not_integral) << T;
@@ -15542,7 +15656,7 @@ static bool findOMPAllocatorHandleT(Sema &S, SourceLocation Loc,
     }
     Res = S.PerformImplicitConversion(Res.get(), AllocatorHandleEnumTy,
                                       Sema::AA_Initializing,
-                                      /* AllowExplicit */ true);
+                                      /*AllowExplicit=*/true);
     if (!Res.isUsable()) {
       ErrorFound = true;
       break;
@@ -17645,7 +17759,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPLastprivateClause(
     if (AssignmentOp.isInvalid())
       continue;
     AssignmentOp = SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), ELoc,
-                                               /*DiscardedValue*/ false);
+                                               /*DiscardedValue=*/false);
     if (AssignmentOp.isInvalid())
       continue;
 
@@ -18362,7 +18476,7 @@ static bool actOnOMPReductionKindClause(
     // A list item that appears in a reduction clause must not be
     // const-qualified.
     if (rejectConstNotMutableType(S, D, Type, ClauseKind, ELoc,
-                                  /*AcceptIfMutable*/ false, ASE || OASE))
+                                  /*AcceptIfMutable=*/false, ASE || OASE))
       continue;
 
     OpenMPDirectiveKind CurrDir = Stack->getCurrentDirective();
@@ -18755,7 +18869,7 @@ static bool actOnOMPReductionKindClause(
       }
       if (ReductionOp.isUsable())
         ReductionOp = S.ActOnFinishFullExpr(ReductionOp.get(),
-                                            /*DiscardedValue*/ false);
+                                            /*DiscardedValue=*/false);
       if (!ReductionOp.isUsable())
         continue;
     }
@@ -19174,7 +19288,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPLinearClause(
     ExprResult CalcStep = SemaRef.BuildBinOp(
         SemaRef.getCurScope(), StepLoc, BO_Assign, SaveRef.get(), StepExpr);
     CalcStep =
-        SemaRef.ActOnFinishFullExpr(CalcStep.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(CalcStep.get(), /*DiscardedValue=*/false);
 
     // Warn about zero linear step (it would be probably better specified as
     // making corresponding variables 'const').
@@ -19263,7 +19377,7 @@ static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
     else
       Update = *CurPrivate;
     Update = SemaRef.ActOnFinishFullExpr(Update.get(), DE->getBeginLoc(),
-                                         /*DiscardedValue*/ false);
+                                         /*DiscardedValue=*/false);
 
     // Build final: Var = PrivCopy;
     ExprResult Final;
@@ -19274,7 +19388,7 @@ static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
     else
       Final = *CurPrivate;
     Final = SemaRef.ActOnFinishFullExpr(Final.get(), DE->getBeginLoc(),
-                                        /*DiscardedValue*/ false);
+                                        /*DiscardedValue=*/false);
 
     if (!Update.isUsable() || !Final.isUsable()) {
       Updates.push_back(nullptr);
@@ -19453,7 +19567,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPCopyinClause(ArrayRef<Expr *> VarList,
       continue;
     AssignmentOp =
         SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), DE->getExprLoc(),
-                                    /*DiscardedValue*/ false);
+                                    /*DiscardedValue=*/false);
     if (AssignmentOp.isInvalid())
       continue;
 
@@ -19562,7 +19676,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
     if (AssignmentOp.isInvalid())
       continue;
     AssignmentOp = SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), ELoc,
-                                               /*DiscardedValue*/ false);
+                                               /*DiscardedValue=*/false);
     if (AssignmentOp.isInvalid())
       continue;
 
@@ -20948,7 +21062,7 @@ static ExprResult buildImplicitMapper(Sema &S, QualType BaseType,
                                            BaseType, MapperId, Maps, nullptr);
   Scope *Scope = S.getScopeForContext(DCT);
   if (Scope)
-    S.PushOnScopeChains(DMD, Scope, /*AddToContext*/ false);
+    S.PushOnScopeChains(DMD, Scope, /*AddToContext=*/false);
   DCT->addDecl(DMD);
   DMD->setAccess(clang::AS_none);
   auto *VD = cast<DeclRefExpr>(MapperVarRef)->getDecl();
diff --git a/clang/lib/Sema/SemaPPC.cpp b/clang/lib/Sema/SemaPPC.cpp
index e0a978e6cf5b97c..b6802d6890ed177 100644
--- a/clang/lib/Sema/SemaPPC.cpp
+++ b/clang/lib/Sema/SemaPPC.cpp
@@ -19,6 +19,7 @@
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Sema.h"
 #include "llvm/ADT/APSInt.h"
 
diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index 2b55c598d55c038..f2d13d456c25fc6 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -10,6 +10,7 @@
 
 #include "clang/Sema/SemaSYCL.h"
 #include "clang/AST/Mangle.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/Sema/Attr.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index d283eaa511011b3..ba681671eb3290f 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -281,7 +281,8 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S, unsigned DiagID) {
   E = WarnExpr;
   if (const auto *Cast = dyn_cast<CastExpr>(E))
     if (Cast->getCastKind() == CK_NoOp ||
-        Cast->getCastKind() == CK_ConstructorConversion)
+        Cast->getCastKind() == CK_ConstructorConversion ||
+        Cast->getCastKind() == CK_IntegralCast)
       E = Cast->getSubExpr()->IgnoreImpCasts();
 
   if (const CallExpr *CE = dyn_cast<CallExpr>(E)) {
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 25585f683752ac4..0afe6064bab185e 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -10124,8 +10124,9 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
   //  instantiated from the member definition associated with its class
   //  template.
   UnresolvedSet<8> TemplateMatches;
-  FunctionDecl *NonTemplateMatch = nullptr;
-  TemplateSpecCandidateSet FailedCandidates(D.getIdentifierLoc());
+  OverloadCandidateSet NonTemplateMatches(D.getBeginLoc(),
+                                          OverloadCandidateSet::CSK_Normal);
+  TemplateSpecCandidateSet FailedTemplateCandidates(D.getIdentifierLoc());
   for (LookupResult::iterator P = Previous.begin(), PEnd = Previous.end();
        P != PEnd; ++P) {
     NamedDecl *Prev = *P;
@@ -10137,9 +10138,18 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
           if (Method->getPrimaryTemplate()) {
             TemplateMatches.addDecl(Method, P.getAccess());
           } else {
-            // FIXME: Can this assert ever happen?  Needs a test.
-            assert(!NonTemplateMatch && "Multiple NonTemplateMatches");
-            NonTemplateMatch = Method;
+            OverloadCandidate &C = NonTemplateMatches.addCandidate();
+            C.FoundDecl = P.getPair();
+            C.Function = Method;
+            C.Viable = true;
+            ConstraintSatisfaction S;
+            if (Method->getTrailingRequiresClause() &&
+                (CheckFunctionConstraints(Method, S, D.getIdentifierLoc(),
+                                          /*ForOverloadResolution=*/true) ||
+                 !S.IsSatisfied)) {
+              C.Viable = false;
+              C.FailureKind = ovl_fail_constraints_not_satisfied;
+            }
           }
         }
       }
@@ -10149,16 +10159,16 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     if (!FunTmpl)
       continue;
 
-    TemplateDeductionInfo Info(FailedCandidates.getLocation());
+    TemplateDeductionInfo Info(FailedTemplateCandidates.getLocation());
     FunctionDecl *Specialization = nullptr;
     if (TemplateDeductionResult TDK = DeduceTemplateArguments(
             FunTmpl, (HasExplicitTemplateArgs ? &TemplateArgs : nullptr), R,
             Specialization, Info);
         TDK != TemplateDeductionResult::Success) {
       // Keep track of almost-matches.
-      FailedCandidates.addCandidate()
-          .set(P.getPair(), FunTmpl->getTemplatedDecl(),
-               MakeDeductionFailureInfo(Context, TDK, Info));
+      FailedTemplateCandidates.addCandidate().set(
+          P.getPair(), FunTmpl->getTemplatedDecl(),
+          MakeDeductionFailureInfo(Context, TDK, Info));
       (void)TDK;
       continue;
     }
@@ -10172,7 +10182,7 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
         CUDA().IdentifyTarget(Specialization,
                               /* IgnoreImplicitHDAttr = */ true) !=
             CUDA().IdentifyTarget(D.getDeclSpec().getAttributes())) {
-      FailedCandidates.addCandidate().set(
+      FailedTemplateCandidates.addCandidate().set(
           P.getPair(), FunTmpl->getTemplatedDecl(),
           MakeDeductionFailureInfo(
               Context, TemplateDeductionResult::CUDATargetMismatch, Info));
@@ -10182,12 +10192,40 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     TemplateMatches.addDecl(Specialization, P.getAccess());
   }
 
-  FunctionDecl *Specialization = NonTemplateMatch;
+  FunctionDecl *Specialization = nullptr;
+  if (!NonTemplateMatches.empty()) {
+    unsigned Msg = 0;
+    OverloadCandidateDisplayKind DisplayKind;
+    OverloadCandidateSet::iterator Best;
+    switch (NonTemplateMatches.BestViableFunction(*this, D.getIdentifierLoc(),
+                                                  Best)) {
+    case OR_Success:
+    case OR_Deleted:
+      Specialization = cast<FunctionDecl>(Best->Function);
+      break;
+    case OR_Ambiguous:
+      Msg = diag::err_explicit_instantiation_ambiguous;
+      DisplayKind = OCD_AmbiguousCandidates;
+      break;
+    case OR_No_Viable_Function:
+      Msg = diag::err_explicit_instantiation_no_candidate;
+      DisplayKind = OCD_AllCandidates;
+      break;
+    }
+    if (Msg) {
+      PartialDiagnostic Diag = PDiag(Msg) << Name;
+      NonTemplateMatches.NoteCandidates(
+          PartialDiagnosticAt(D.getIdentifierLoc(), Diag), *this, DisplayKind,
+          {});
+      return true;
+    }
+  }
+
   if (!Specialization) {
     // Find the most specialized function template specialization.
     UnresolvedSetIterator Result = getMostSpecialized(
-        TemplateMatches.begin(), TemplateMatches.end(), FailedCandidates,
-        D.getIdentifierLoc(),
+        TemplateMatches.begin(), TemplateMatches.end(),
+        FailedTemplateCandidates, D.getIdentifierLoc(),
         PDiag(diag::err_explicit_instantiation_not_known) << Name,
         PDiag(diag::err_explicit_instantiation_ambiguous) << Name,
         PDiag(diag::note_explicit_instantiation_candidate));
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index ec951d5ac06dbc6..35bc8cd713f06c6 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -28,6 +28,7 @@
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/AST/UnresolvedSet.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/ExceptionSpecificationType.h"
@@ -4462,6 +4463,9 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
       ParamTypesForArgChecking.push_back(ParamType);
 
       if (ParamIdx == 0 && HasExplicitObject) {
+        if (ObjectType.isNull())
+          return TemplateDeductionResult::InvalidExplicitArguments;
+
         if (auto Result = DeduceCallArgument(ParamType, 0,
                                              /*ExplicitObjectArgument=*/true);
             Result != TemplateDeductionResult::Success)
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index f93cd113988ae42..a58854acb21fa5b 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -1442,8 +1443,47 @@ Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
     if (D->isUnsupportedFriend()) {
       InstTy = Ty;
     } else {
-      InstTy = SemaRef.SubstType(Ty, TemplateArgs,
-                                 D->getLocation(), DeclarationName());
+      if (D->isPackExpansion()) {
+        SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+        SemaRef.collectUnexpandedParameterPacks(Ty->getTypeLoc(), Unexpanded);
+        assert(!Unexpanded.empty() && "Pack expansion without packs");
+
+        bool ShouldExpand = true;
+        bool RetainExpansion = false;
+        std::optional<unsigned> NumExpansions;
+        if (SemaRef.CheckParameterPacksForExpansion(
+                D->getEllipsisLoc(), D->getSourceRange(), Unexpanded,
+                TemplateArgs, ShouldExpand, RetainExpansion, NumExpansions))
+          return nullptr;
+
+        assert(!RetainExpansion &&
+               "should never retain an expansion for a variadic friend decl");
+
+        if (ShouldExpand) {
+          SmallVector<FriendDecl *> Decls;
+          for (unsigned I = 0; I != *NumExpansions; I++) {
+            Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, I);
+            TypeSourceInfo *TSI = SemaRef.SubstType(
+                Ty, TemplateArgs, D->getEllipsisLoc(), DeclarationName());
+            if (!TSI)
+              return nullptr;
+
+            auto FD =
+                FriendDecl::Create(SemaRef.Context, Owner, D->getLocation(),
+                                   TSI, D->getFriendLoc());
+
+            FD->setAccess(AS_public);
+            Owner->addDecl(FD);
+            Decls.push_back(FD);
+          }
+
+          // Just drop this node; we have no use for it anymore.
+          return nullptr;
+        }
+      }
+
+      InstTy = SemaRef.SubstType(Ty, TemplateArgs, D->getLocation(),
+                                 DeclarationName());
     }
     if (!InstTy)
       return nullptr;
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 08020f9f889270a..4b1bb05464f7dca 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -20,6 +20,8 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprObjC.h"
+#include "clang/AST/LocInfoType.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeLocVisitor.h"
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index a0756f167deae61..45c4a1c80b40835 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -13,6 +13,7 @@
 #include "clang/Sema/SemaX86.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Attr.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index a9199f7e50f5dc8..ef160228933c598 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2361,6 +2361,7 @@ void ASTDeclReader::VisitFriendDecl(FriendDecl *D) {
   D->NextFriend = readDeclID().getRawValue();
   D->UnsupportedFriend = (Record.readInt() != 0);
   D->FriendLoc = readSourceLocation();
+  D->EllipsisLoc = readSourceLocation();
 }
 
 void ASTDeclReader::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 8ae07907a04abaa..a33f2a41a654974 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -1128,7 +1128,6 @@ void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) {
       (BinaryOperator::Opcode)CurrentUnpackingBits->getNextBits(/*Width=*/6));
   bool hasFP_Features = CurrentUnpackingBits->getNextBit();
   E->setHasStoredFPFeatures(hasFP_Features);
-  E->setExcludedOverflowPattern(CurrentUnpackingBits->getNextBit());
   E->setLHS(Record.readSubExpr());
   E->setRHS(Record.readSubExpr());
   E->setOperatorLoc(readSourceLocation());
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 8a4ca54349e38f1..555f6325da646bf 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1660,6 +1660,7 @@ void ASTDeclWriter::VisitFriendDecl(FriendDecl *D) {
   Record.AddDeclRef(D->getNextFriend());
   Record.push_back(D->UnsupportedFriend);
   Record.AddSourceLocation(D->FriendLoc);
+  Record.AddSourceLocation(D->EllipsisLoc);
   Code = serialization::DECL_FRIEND;
 }
 
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index c292d0a789c7cd9..038616a675b7271 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -1063,7 +1063,6 @@ void ASTStmtWriter::VisitBinaryOperator(BinaryOperator *E) {
   CurrentPackingBits.addBits(E->getOpcode(), /*Width=*/6);
   bool HasFPFeatures = E->hasStoredFPFeatures();
   CurrentPackingBits.addBit(HasFPFeatures);
-  CurrentPackingBits.addBit(E->hasExcludedOverflowPattern());
   Record.AddStmt(E->getLHS());
   Record.AddStmt(E->getRHS());
   Record.AddSourceLocation(E->getOperatorLoc());
diff --git a/clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp
index 8f1c31763e212c3..f50b5cd3c9e497e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp
@@ -48,7 +48,7 @@ bool StringChecker::isCharToStringCtor(const CallEvent &Call,
 
   // Verify that the parameters have the expected types:
   // - arg 1: `const CharT *`
-  // - arg 2: some allocator - which is definately not `size_t`.
+  // - arg 2: some allocator - which is definitely not `size_t`.
   const QualType Arg1Ty = Call.getArgExpr(0)->getType().getCanonicalType();
   const QualType Arg2Ty = Call.getArgExpr(1)->getType().getCanonicalType();
 
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index 686310d38ebd58f..493914f88ba9a50 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -3807,6 +3807,14 @@ void ExprEngine::VisitGCCAsmStmt(const GCCAsmStmt *A, ExplodedNode *Pred,
       state = state->bindLoc(*LV, UnknownVal(), Pred->getLocationContext());
   }
 
+  // Do not reason about locations passed inside inline assembly.
+  for (const Expr *I : A->inputs()) {
+    SVal X = state->getSVal(I, Pred->getLocationContext());
+
+    if (std::optional<Loc> LV = X.getAs<Loc>())
+      state = state->bindLoc(*LV, UnknownVal(), Pred->getLocationContext());
+  }
+
   Bldr.generateNode(A, Pred, state);
 }
 
diff --git a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index fb5030d373c2f18..28d9de2973b01fa 100644
--- a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -27,6 +27,7 @@
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallString.h"
@@ -52,6 +53,7 @@
 
 using namespace clang;
 using namespace ento;
+using llvm::RewriteBuffer;
 
 //===----------------------------------------------------------------------===//
 // Boilerplate.
diff --git a/clang/lib/Tooling/Core/Replacement.cpp b/clang/lib/Tooling/Core/Replacement.cpp
index 269f17a6db4cfc1..9acf00263b5a8ce 100644
--- a/clang/lib/Tooling/Core/Replacement.cpp
+++ b/clang/lib/Tooling/Core/Replacement.cpp
@@ -19,9 +19,9 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Lexer.h"
-#include "clang/Rewrite/Core/RewriteBuffer.h"
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
diff --git a/clang/test/APINotes/Inputs/Headers/Fields.apinotes b/clang/test/APINotes/Inputs/Headers/Fields.apinotes
new file mode 100644
index 000000000000000..931da52ba29d183
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/Fields.apinotes
@@ -0,0 +1,23 @@
+---
+Name: Fields
+Tags:
+- Name: IntWrapper
+  Fields:
+  - Name: value
+    Availability: none
+    AvailabilityMsg: "oh no"
+- Name: Outer
+  Tags:
+  - Name: Inner
+    Fields:
+    - Name: value
+      Availability: none
+      AvailabilityMsg: "oh no 2"
+  Methods:
+  - Name: value
+    Availability: none
+    AvailabilityMsg: "this should have no effect"  
+Functions:
+- Name: value
+  Availability: none
+  AvailabilityMsg: "this should have no effect"
diff --git a/clang/test/APINotes/Inputs/Headers/Fields.h b/clang/test/APINotes/Inputs/Headers/Fields.h
new file mode 100644
index 000000000000000..dbd342da4778922
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/Fields.h
@@ -0,0 +1,21 @@
+struct IntWrapper {
+  int value;
+
+  bool : 1;
+
+  enum {
+    one,
+    two,
+    three
+  };
+  union {
+    int a;
+    char b;
+  };
+};
+
+struct Outer {
+  struct Inner {
+    int value;
+  };
+};
diff --git a/clang/test/APINotes/Inputs/Headers/module.modulemap b/clang/test/APINotes/Inputs/Headers/module.modulemap
index faf6042c78d576e..7d304863398ecfc 100644
--- a/clang/test/APINotes/Inputs/Headers/module.modulemap
+++ b/clang/test/APINotes/Inputs/Headers/module.modulemap
@@ -12,6 +12,11 @@ module ExportAs {
   export *
 }
 
+module Fields {
+  header "Fields.h"
+  export *
+}
+
 module HeaderLib {
   header "HeaderLib.h"
 }
diff --git a/clang/test/APINotes/fields.cpp b/clang/test/APINotes/fields.cpp
new file mode 100644
index 000000000000000..8dea2229b4e0a27
--- /dev/null
+++ b/clang/test/APINotes/fields.cpp
@@ -0,0 +1,17 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Fields -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -x c++
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Fields -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter IntWrapper::value -x c++ | FileCheck --check-prefix=CHECK-FIELD %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Fields -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Outer::Inner::value -x c++ | FileCheck --check-prefix=CHECK-DEEP-FIELD %s
+
+#include "Fields.h"
+
+// CHECK-FIELD: Dumping IntWrapper::value:
+// CHECK-FIELD-NEXT: FieldDecl {{.+}} value
+// CHECK-FIELD: UnavailableAttr {{.+}} <<invalid sloc>> "oh no"
+
+// CHECK-DEEP-FIELD: Dumping Outer::Inner::value:
+// CHECK-DEEP-FIELD-NEXT: FieldDecl {{.+}} value
+// CHECK-DEEP-FIELD: UnavailableAttr {{.+}} <<invalid sloc>> "oh no 2"
+
+// CHECK-FIELD-NOT: this should have no effect
+// CHECK-DEEP-FIELD-NOT: this should have no effect
diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/ByteCode/arrays.cpp
similarity index 100%
rename from clang/test/AST/Interp/arrays.cpp
rename to clang/test/AST/ByteCode/arrays.cpp
diff --git a/clang/test/AST/Interp/atomic.c b/clang/test/AST/ByteCode/atomic.c
similarity index 100%
rename from clang/test/AST/Interp/atomic.c
rename to clang/test/AST/ByteCode/atomic.c
diff --git a/clang/test/AST/Interp/atomic.cpp b/clang/test/AST/ByteCode/atomic.cpp
similarity index 100%
rename from clang/test/AST/Interp/atomic.cpp
rename to clang/test/AST/ByteCode/atomic.cpp
diff --git a/clang/test/AST/Interp/bitfields.cpp b/clang/test/AST/ByteCode/bitfields.cpp
similarity index 100%
rename from clang/test/AST/Interp/bitfields.cpp
rename to clang/test/AST/ByteCode/bitfields.cpp
diff --git a/clang/test/AST/Interp/builtin-align-cxx.cpp b/clang/test/AST/ByteCode/builtin-align-cxx.cpp
similarity index 100%
rename from clang/test/AST/Interp/builtin-align-cxx.cpp
rename to clang/test/AST/ByteCode/builtin-align-cxx.cpp
diff --git a/clang/test/AST/Interp/builtin-constant-p.cpp b/clang/test/AST/ByteCode/builtin-constant-p.cpp
similarity index 100%
rename from clang/test/AST/Interp/builtin-constant-p.cpp
rename to clang/test/AST/ByteCode/builtin-constant-p.cpp
diff --git a/clang/test/AST/Interp/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
similarity index 100%
rename from clang/test/AST/Interp/builtin-functions.cpp
rename to clang/test/AST/ByteCode/builtin-functions.cpp
diff --git a/clang/test/AST/Interp/builtins.cpp b/clang/test/AST/ByteCode/builtins.cpp
similarity index 100%
rename from clang/test/AST/Interp/builtins.cpp
rename to clang/test/AST/ByteCode/builtins.cpp
diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/ByteCode/c.c
similarity index 99%
rename from clang/test/AST/Interp/c.c
rename to clang/test/AST/ByteCode/c.c
index 13a5e082a125f20..b38259d41130eb1 100644
--- a/clang/test/AST/Interp/c.c
+++ b/clang/test/AST/ByteCode/c.c
@@ -295,4 +295,5 @@ void T1(void) {
                                                            // pedantic-warning {{use of GNU statement expression extension}}
 }
 
-
+enum teste1 test1f(void), (*test1)(void) = test1f; // pedantic-warning {{ISO C forbids forward references to 'enum' types}}
+enum teste1 { TEST1 };
diff --git a/clang/test/AST/Interp/c23.c b/clang/test/AST/ByteCode/c23.c
similarity index 100%
rename from clang/test/AST/Interp/c23.c
rename to clang/test/AST/ByteCode/c23.c
diff --git a/clang/test/AST/ByteCode/codegen.c b/clang/test/AST/ByteCode/codegen.c
new file mode 100644
index 000000000000000..8434992823010e7
--- /dev/null
+++ b/clang/test/AST/ByteCode/codegen.c
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s -fexperimental-new-constant-interpreter | FileCheck %s
+
+typedef __INTPTR_TYPE__ intptr_t;
+
+const intptr_t Z1 = (intptr_t)(((char*)-1LL) + 1);
+// CHECK: @Z1 = constant i64 0
+
+const intptr_t Z2 = (intptr_t)(((char*)1LL) - 1);
+// CHECK: @Z2 = constant i64 0
+
+struct A {
+  char num_fields;
+};
+struct B {
+  char a, b[1];
+};
+const int A = (char *)(&( (struct B *)(16) )->b[0]) - (char *)(16);
+// CHECK: @A = constant i32 1
diff --git a/clang/test/AST/Interp/codegen.cpp b/clang/test/AST/ByteCode/codegen.cpp
similarity index 91%
rename from clang/test/AST/Interp/codegen.cpp
rename to clang/test/AST/ByteCode/codegen.cpp
index 920a5ad7ba3e8fc..9fac28a52d3150d 100644
--- a/clang/test/AST/Interp/codegen.cpp
+++ b/clang/test/AST/ByteCode/codegen.cpp
@@ -64,3 +64,12 @@ namespace Bitfield {
   // CHECK: store ptr @_ZGRN8Bitfield1rE_, ptr @_ZN8Bitfield1rE, align 8
   int &&r = S().a;
 }
+
+namespace Null {
+  decltype(nullptr) null();
+  // CHECK: call {{.*}} @_ZN4Null4nullEv(
+  int *p = null();
+  struct S {};
+  // CHECK: call {{.*}} @_ZN4Null4nullEv(
+  int S::*q = null();
+}
diff --git a/clang/test/AST/Interp/comma.cpp b/clang/test/AST/ByteCode/comma.cpp
similarity index 100%
rename from clang/test/AST/Interp/comma.cpp
rename to clang/test/AST/ByteCode/comma.cpp
diff --git a/clang/test/AST/Interp/complex.c b/clang/test/AST/ByteCode/complex.c
similarity index 100%
rename from clang/test/AST/Interp/complex.c
rename to clang/test/AST/ByteCode/complex.c
diff --git a/clang/test/AST/Interp/complex.cpp b/clang/test/AST/ByteCode/complex.cpp
similarity index 100%
rename from clang/test/AST/Interp/complex.cpp
rename to clang/test/AST/ByteCode/complex.cpp
diff --git a/clang/test/AST/Interp/cond.cpp b/clang/test/AST/ByteCode/cond.cpp
similarity index 100%
rename from clang/test/AST/Interp/cond.cpp
rename to clang/test/AST/ByteCode/cond.cpp
diff --git a/clang/test/AST/Interp/const-eval.c b/clang/test/AST/ByteCode/const-eval.c
similarity index 100%
rename from clang/test/AST/Interp/const-eval.c
rename to clang/test/AST/ByteCode/const-eval.c
diff --git a/clang/test/AST/Interp/const-fpfeatures.cpp b/clang/test/AST/ByteCode/const-fpfeatures.cpp
similarity index 100%
rename from clang/test/AST/Interp/const-fpfeatures.cpp
rename to clang/test/AST/ByteCode/const-fpfeatures.cpp
diff --git a/clang/test/AST/Interp/const-temporaries.cpp b/clang/test/AST/ByteCode/const-temporaries.cpp
similarity index 100%
rename from clang/test/AST/Interp/const-temporaries.cpp
rename to clang/test/AST/ByteCode/const-temporaries.cpp
diff --git a/clang/test/AST/Interp/constexpr-frame-describe.cpp b/clang/test/AST/ByteCode/constexpr-frame-describe.cpp
similarity index 100%
rename from clang/test/AST/Interp/constexpr-frame-describe.cpp
rename to clang/test/AST/ByteCode/constexpr-frame-describe.cpp
diff --git a/clang/test/AST/Interp/constexpr-nqueens.cpp b/clang/test/AST/ByteCode/constexpr-nqueens.cpp
similarity index 100%
rename from clang/test/AST/Interp/constexpr-nqueens.cpp
rename to clang/test/AST/ByteCode/constexpr-nqueens.cpp
diff --git a/clang/test/AST/Interp/constexpr-subobj-initialization.cpp b/clang/test/AST/ByteCode/constexpr-subobj-initialization.cpp
similarity index 100%
rename from clang/test/AST/Interp/constexpr-subobj-initialization.cpp
rename to clang/test/AST/ByteCode/constexpr-subobj-initialization.cpp
diff --git a/clang/test/AST/Interp/crash-GH49103-2.cpp b/clang/test/AST/ByteCode/crash-GH49103-2.cpp
similarity index 100%
rename from clang/test/AST/Interp/crash-GH49103-2.cpp
rename to clang/test/AST/ByteCode/crash-GH49103-2.cpp
diff --git a/clang/test/AST/Interp/cxx03.cpp b/clang/test/AST/ByteCode/cxx03.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx03.cpp
rename to clang/test/AST/ByteCode/cxx03.cpp
diff --git a/clang/test/AST/Interp/cxx11.cpp b/clang/test/AST/ByteCode/cxx11.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx11.cpp
rename to clang/test/AST/ByteCode/cxx11.cpp
diff --git a/clang/test/AST/Interp/cxx17.cpp b/clang/test/AST/ByteCode/cxx17.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx17.cpp
rename to clang/test/AST/ByteCode/cxx17.cpp
diff --git a/clang/test/AST/Interp/cxx20.cpp b/clang/test/AST/ByteCode/cxx20.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx20.cpp
rename to clang/test/AST/ByteCode/cxx20.cpp
diff --git a/clang/test/AST/Interp/cxx23.cpp b/clang/test/AST/ByteCode/cxx23.cpp
similarity index 95%
rename from clang/test/AST/Interp/cxx23.cpp
rename to clang/test/AST/ByteCode/cxx23.cpp
index eb05a9fda0dfb13..756eec5b8256053 100644
--- a/clang/test/AST/Interp/cxx23.cpp
+++ b/clang/test/AST/ByteCode/cxx23.cpp
@@ -224,3 +224,17 @@ namespace ExplicitLambdaInstancePointer {
       static_assert(fp(1) == 1, "");
   }
 }
+
+namespace TwosComplementShifts {
+  using uint32 = __UINT32_TYPE__;
+  using int32 = __INT32_TYPE__;
+  static_assert(uint32(int32(0x1234) << 16) == 0x12340000);
+  static_assert(uint32(int32(0x1234) << 19) == 0x91a00000);
+  static_assert(uint32(int32(0x1234) << 20) == 0x23400000);
+  static_assert(uint32(int32(0x1234) << 24) == 0x34000000);
+  static_assert(uint32(int32(-1) << 31) == 0x80000000);
+
+  static_assert(-2 >> 1 == -1);
+  static_assert(-3 >> 1 == -2);
+  static_assert(-7 >> 1 == -4);
+}
diff --git a/clang/test/AST/Interp/cxx26.cpp b/clang/test/AST/ByteCode/cxx26.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx26.cpp
rename to clang/test/AST/ByteCode/cxx26.cpp
diff --git a/clang/test/AST/Interp/cxx2a.cpp b/clang/test/AST/ByteCode/cxx2a.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx2a.cpp
rename to clang/test/AST/ByteCode/cxx2a.cpp
diff --git a/clang/test/AST/Interp/cxx98.cpp b/clang/test/AST/ByteCode/cxx98.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx98.cpp
rename to clang/test/AST/ByteCode/cxx98.cpp
diff --git a/clang/test/AST/Interp/depth-limit.cpp b/clang/test/AST/ByteCode/depth-limit.cpp
similarity index 100%
rename from clang/test/AST/Interp/depth-limit.cpp
rename to clang/test/AST/ByteCode/depth-limit.cpp
diff --git a/clang/test/AST/Interp/depth-limit2.cpp b/clang/test/AST/ByteCode/depth-limit2.cpp
similarity index 100%
rename from clang/test/AST/Interp/depth-limit2.cpp
rename to clang/test/AST/ByteCode/depth-limit2.cpp
diff --git a/clang/test/AST/Interp/enums-targets.cpp b/clang/test/AST/ByteCode/enums-targets.cpp
similarity index 100%
rename from clang/test/AST/Interp/enums-targets.cpp
rename to clang/test/AST/ByteCode/enums-targets.cpp
diff --git a/clang/test/AST/Interp/enums.cpp b/clang/test/AST/ByteCode/enums.cpp
similarity index 90%
rename from clang/test/AST/Interp/enums.cpp
rename to clang/test/AST/ByteCode/enums.cpp
index c4db7875451677e..459cf3a0c83d540 100644
--- a/clang/test/AST/Interp/enums.cpp
+++ b/clang/test/AST/ByteCode/enums.cpp
@@ -48,3 +48,8 @@ constexpr EC getB() {
 
 
 static_assert(getB() == EC::B, "");
+
+namespace B {
+  enum E : bool { Zero, One };
+  static_assert((int)(E)2 == 1, "");
+} // namespace B
diff --git a/clang/test/AST/Interp/eval-order.cpp b/clang/test/AST/ByteCode/eval-order.cpp
similarity index 100%
rename from clang/test/AST/Interp/eval-order.cpp
rename to clang/test/AST/ByteCode/eval-order.cpp
diff --git a/clang/test/AST/Interp/floats.cpp b/clang/test/AST/ByteCode/floats.cpp
similarity index 100%
rename from clang/test/AST/Interp/floats.cpp
rename to clang/test/AST/ByteCode/floats.cpp
diff --git a/clang/test/AST/Interp/functions.cpp b/clang/test/AST/ByteCode/functions.cpp
similarity index 96%
rename from clang/test/AST/Interp/functions.cpp
rename to clang/test/AST/ByteCode/functions.cpp
index f190262ad3ebee4..b00f59a8d8d433d 100644
--- a/clang/test/AST/Interp/functions.cpp
+++ b/clang/test/AST/ByteCode/functions.cpp
@@ -221,6 +221,17 @@ namespace Comparison {
   static_assert(pg == &g, "");
 }
 
+  constexpr int Double(int n) { return 2 * n; }
+  constexpr int Triple(int n) { return 3 * n; }
+  constexpr int Twice(int (*F)(int), int n) { return F(F(n)); }
+  constexpr int Quadruple(int n) { return Twice(Double, n); }
+  constexpr auto Select(int n) -> int (*)(int) {
+    return n == 2 ? &Double : n == 3 ? &Triple : n == 4 ? &Quadruple : 0;
+  }
+  constexpr int Apply(int (*F)(int), int n) { return F(n); } // both-note {{'F' evaluates to a null function pointer}}
+
+  constexpr int Invalid = Apply(Select(0), 0); // both-error {{must be initialized by a constant expression}} \
+                                               // both-note {{in call to 'Apply(nullptr, 0)'}}
 }
 
 struct F {
diff --git a/clang/test/AST/Interp/hlsl.hlsl b/clang/test/AST/ByteCode/hlsl.hlsl
similarity index 100%
rename from clang/test/AST/Interp/hlsl.hlsl
rename to clang/test/AST/ByteCode/hlsl.hlsl
diff --git a/clang/test/AST/Interp/if.cpp b/clang/test/AST/ByteCode/if.cpp
similarity index 100%
rename from clang/test/AST/Interp/if.cpp
rename to clang/test/AST/ByteCode/if.cpp
diff --git a/clang/test/AST/Interp/intap.cpp b/clang/test/AST/ByteCode/intap.cpp
similarity index 100%
rename from clang/test/AST/Interp/intap.cpp
rename to clang/test/AST/ByteCode/intap.cpp
diff --git a/clang/test/AST/Interp/invalid.cpp b/clang/test/AST/ByteCode/invalid.cpp
similarity index 100%
rename from clang/test/AST/Interp/invalid.cpp
rename to clang/test/AST/ByteCode/invalid.cpp
diff --git a/clang/test/AST/Interp/lambda.cpp b/clang/test/AST/ByteCode/lambda.cpp
similarity index 100%
rename from clang/test/AST/Interp/lambda.cpp
rename to clang/test/AST/ByteCode/lambda.cpp
diff --git a/clang/test/AST/Interp/lifetimes.cpp b/clang/test/AST/ByteCode/lifetimes.cpp
similarity index 93%
rename from clang/test/AST/Interp/lifetimes.cpp
rename to clang/test/AST/ByteCode/lifetimes.cpp
index 9fca54fe11120aa..9a99485c4a40bff 100644
--- a/clang/test/AST/Interp/lifetimes.cpp
+++ b/clang/test/AST/ByteCode/lifetimes.cpp
@@ -60,3 +60,11 @@ namespace MoveFnWorks {
   }
   static_assert(dtor_calls_dtor(), "");
 }
+
+namespace PrimitiveMoveFn {
+  /// This used to crash.
+  void test() {
+    const float y = 100;
+    const float &x = y;
+  }
+}
diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/ByteCode/literals.cpp
similarity index 100%
rename from clang/test/AST/Interp/literals.cpp
rename to clang/test/AST/ByteCode/literals.cpp
diff --git a/clang/test/AST/Interp/loops.cpp b/clang/test/AST/ByteCode/loops.cpp
similarity index 100%
rename from clang/test/AST/Interp/loops.cpp
rename to clang/test/AST/ByteCode/loops.cpp
diff --git a/clang/test/AST/Interp/memberpointers.cpp b/clang/test/AST/ByteCode/memberpointers.cpp
similarity index 94%
rename from clang/test/AST/Interp/memberpointers.cpp
rename to clang/test/AST/ByteCode/memberpointers.cpp
index f38e94863863159..ea4a725d3fd0dfa 100644
--- a/clang/test/AST/Interp/memberpointers.cpp
+++ b/clang/test/AST/ByteCode/memberpointers.cpp
@@ -209,3 +209,20 @@ namespace MemPtrTemporary {
 
   static_assert(apply(A(), &A::f) == 5, "");
 }
+
+namespace IndirectFields {
+  struct I { union { struct { int a, b; }; }; };
+
+  template <typename T, int T::*F>
+  constexpr int ReadField(const T &o) {
+    return F ? o.*F : 0;
+  }
+  void ReadFields() {
+    I i;
+    ReadField<I, &I::a>(i);
+    ReadField<I, &I::b>(i);
+  }
+
+  constexpr I i{12};
+  static_assert(ReadField<I, &I::a>(i) == 12, "");
+}
diff --git a/clang/test/AST/Interp/ms.cpp b/clang/test/AST/ByteCode/ms.cpp
similarity index 100%
rename from clang/test/AST/Interp/ms.cpp
rename to clang/test/AST/ByteCode/ms.cpp
diff --git a/clang/test/AST/Interp/mutable.cpp b/clang/test/AST/ByteCode/mutable.cpp
similarity index 100%
rename from clang/test/AST/Interp/mutable.cpp
rename to clang/test/AST/ByteCode/mutable.cpp
diff --git a/clang/test/AST/Interp/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
similarity index 100%
rename from clang/test/AST/Interp/new-delete.cpp
rename to clang/test/AST/ByteCode/new-delete.cpp
diff --git a/clang/test/AST/Interp/nullable.cpp b/clang/test/AST/ByteCode/nullable.cpp
similarity index 100%
rename from clang/test/AST/Interp/nullable.cpp
rename to clang/test/AST/ByteCode/nullable.cpp
diff --git a/clang/test/AST/Interp/objc.mm b/clang/test/AST/ByteCode/objc.mm
similarity index 100%
rename from clang/test/AST/Interp/objc.mm
rename to clang/test/AST/ByteCode/objc.mm
diff --git a/clang/test/AST/Interp/opencl.cl b/clang/test/AST/ByteCode/opencl.cl
similarity index 100%
rename from clang/test/AST/Interp/opencl.cl
rename to clang/test/AST/ByteCode/opencl.cl
diff --git a/clang/test/AST/Interp/pointer-addition.c b/clang/test/AST/ByteCode/pointer-addition.c
similarity index 100%
rename from clang/test/AST/Interp/pointer-addition.c
rename to clang/test/AST/ByteCode/pointer-addition.c
diff --git a/clang/test/AST/Interp/records.cpp b/clang/test/AST/ByteCode/records.cpp
similarity index 98%
rename from clang/test/AST/Interp/records.cpp
rename to clang/test/AST/ByteCode/records.cpp
index f51f9771b38aaa6..7e3cf5b94518f7c 100644
--- a/clang/test/AST/Interp/records.cpp
+++ b/clang/test/AST/ByteCode/records.cpp
@@ -1627,3 +1627,29 @@ namespace DtorDestroysFieldsAfterSelf {
   static_assert(foo() == 15);
 }
 #endif
+
+namespace ExprWithCleanups {
+  struct A { A(); ~A(); int get(); };
+  constexpr int get() {return false ? A().get() : 1;}
+  static_assert(get() == 1, "");
+
+
+  struct S {
+    int V;
+    constexpr S(int V) : V(V) {}
+    constexpr int get() {
+      return V;
+    }
+  };
+  constexpr int get(bool b) {
+    S a = b ? S(1) : S(2);
+
+    return a.get();
+  }
+  static_assert(get(true) == 1, "");
+  static_assert(get(false) == 2, "");
+
+
+  constexpr auto F = true ? 1i : 2i;
+  static_assert(F == 1i, "");
+}
diff --git a/clang/test/AST/Interp/references.cpp b/clang/test/AST/ByteCode/references.cpp
similarity index 100%
rename from clang/test/AST/Interp/references.cpp
rename to clang/test/AST/ByteCode/references.cpp
diff --git a/clang/test/AST/Interp/shifts.cpp b/clang/test/AST/ByteCode/shifts.cpp
similarity index 100%
rename from clang/test/AST/Interp/shifts.cpp
rename to clang/test/AST/ByteCode/shifts.cpp
diff --git a/clang/test/AST/Interp/spaceship.cpp b/clang/test/AST/ByteCode/spaceship.cpp
similarity index 100%
rename from clang/test/AST/Interp/spaceship.cpp
rename to clang/test/AST/ByteCode/spaceship.cpp
diff --git a/clang/test/AST/Interp/switch.cpp b/clang/test/AST/ByteCode/switch.cpp
similarity index 100%
rename from clang/test/AST/Interp/switch.cpp
rename to clang/test/AST/ByteCode/switch.cpp
diff --git a/clang/test/AST/Interp/sycl.cpp b/clang/test/AST/ByteCode/sycl.cpp
similarity index 100%
rename from clang/test/AST/Interp/sycl.cpp
rename to clang/test/AST/ByteCode/sycl.cpp
diff --git a/clang/test/AST/Interp/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
similarity index 100%
rename from clang/test/AST/Interp/unions.cpp
rename to clang/test/AST/ByteCode/unions.cpp
diff --git a/clang/test/AST/Interp/vectors.cpp b/clang/test/AST/ByteCode/vectors.cpp
similarity index 100%
rename from clang/test/AST/Interp/vectors.cpp
rename to clang/test/AST/ByteCode/vectors.cpp
diff --git a/clang/test/AST/Interp/weak.cpp b/clang/test/AST/ByteCode/weak.cpp
similarity index 100%
rename from clang/test/AST/Interp/weak.cpp
rename to clang/test/AST/ByteCode/weak.cpp
diff --git a/clang/test/AST/ast-dump-funcs-json.cpp b/clang/test/AST/ast-dump-funcs-json.cpp
index 041d98f2713d32e..957df5cea6ec534 100644
--- a/clang/test/AST/ast-dump-funcs-json.cpp
+++ b/clang/test/AST/ast-dump-funcs-json.cpp
@@ -41,13 +41,18 @@ int main() {
   Test1(); // Causes this to be marked 'used'
 }
 
+template <typename ...Ts>
+struct TestFriends {
+    friend Ts...;
+};
+
 // NOTE: CHECK lines have been autogenerated by gen_ast_dump_json_test.py
 
 
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 124,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 4,
 // CHECK-NEXT:   "col": 8,
@@ -55,12 +60,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 119,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 130,
 // CHECK-NEXT:    "col": 14,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -76,7 +81,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 140,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 5,
 // CHECK-NEXT:   "col": 8,
@@ -84,12 +89,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 135,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 148,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 5
 // CHECK-NEXT:   }
@@ -105,7 +110,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 162,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 6,
 // CHECK-NEXT:   "col": 8,
@@ -113,12 +118,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 157,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 170,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 8
 // CHECK-NEXT:   }
@@ -134,7 +139,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 187,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 7,
 // CHECK-NEXT:   "col": 8,
@@ -142,12 +147,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 182,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 195,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -163,7 +168,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 205,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 8,
 // CHECK-NEXT:   "col": 8,
@@ -171,12 +176,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 200,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 213,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 2
 // CHECK-NEXT:   }
@@ -192,7 +197,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 232,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 9,
 // CHECK-NEXT:   "col": 16,
@@ -200,12 +205,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 219,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 7
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 253,
 // CHECK-NEXT:    "col": 37,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -221,18 +226,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 243,
 // CHECK-NEXT:     "col": 27,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 238,
 // CHECK-NEXT:      "col": 22,
 // CHECK-NEXT:      "tokLen": 5
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 238,
 // CHECK-NEXT:      "col": 22,
 // CHECK-NEXT:      "tokLen": 5
 // CHECK-NEXT:     }
@@ -245,18 +250,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 249,
 // CHECK-NEXT:     "col": 33,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 245,
 // CHECK-NEXT:      "col": 29,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 251,
 // CHECK-NEXT:      "col": 35,
 // CHECK-NEXT:      "tokLen": 2
 // CHECK-NEXT:     }
@@ -271,12 +276,12 @@ int main() {
 // CHECK-NEXT:      "kind": "IntegerLiteral",
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 251,
 // CHECK-NEXT:        "col": 35,
 // CHECK-NEXT:        "tokLen": 2
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 251,
 // CHECK-NEXT:        "col": 35,
 // CHECK-NEXT:        "tokLen": 2
 // CHECK-NEXT:       }
@@ -296,7 +301,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 271,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 10,
 // CHECK-NEXT:   "col": 16,
@@ -304,12 +309,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 258,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 7
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 281,
 // CHECK-NEXT:    "col": 26,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -327,7 +332,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 343,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 14,
 // CHECK-NEXT:   "col": 8,
@@ -335,12 +340,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 338,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 367,
 // CHECK-NEXT:    "col": 32,
 // CHECK-NEXT:    "tokLen": 8
 // CHECK-NEXT:   }
@@ -355,18 +360,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 354,
 // CHECK-NEXT:     "col": 19,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 349,
 // CHECK-NEXT:      "col": 14,
 // CHECK-NEXT:      "tokLen": 5
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 349,
 // CHECK-NEXT:      "col": 14,
 // CHECK-NEXT:      "tokLen": 5
 // CHECK-NEXT:     }
@@ -379,18 +384,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 360,
 // CHECK-NEXT:     "col": 25,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 356,
 // CHECK-NEXT:      "col": 21,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 362,
 // CHECK-NEXT:      "col": 27,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     }
@@ -405,12 +410,12 @@ int main() {
 // CHECK-NEXT:      "kind": "IntegerLiteral",
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 362,
 // CHECK-NEXT:        "col": 27,
 // CHECK-NEXT:        "tokLen": 3
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 362,
 // CHECK-NEXT:        "col": 27,
 // CHECK-NEXT:        "tokLen": 3
 // CHECK-NEXT:       }
@@ -428,12 +433,12 @@ int main() {
 // CHECK-NEXT:    "kind": "OverrideAttr",
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 367,
 // CHECK-NEXT:      "col": 32,
 // CHECK-NEXT:      "tokLen": 8
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 367,
 // CHECK-NEXT:      "col": 32,
 // CHECK-NEXT:      "tokLen": 8
 // CHECK-NEXT:     }
@@ -446,7 +451,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 399,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 18,
 // CHECK-NEXT:   "col": 8,
@@ -454,12 +459,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 394,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 405,
 // CHECK-NEXT:    "col": 14,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -475,7 +480,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 419,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 20,
 // CHECK-NEXT:   "col": 9,
@@ -483,12 +488,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 411,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 428,
 // CHECK-NEXT:    "col": 18,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -506,12 +511,12 @@ int main() {
 // CHECK-NEXT:    "kind": "CompoundStmt",
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 427,
 // CHECK-NEXT:      "col": 17,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 428,
 // CHECK-NEXT:      "col": 18,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -524,7 +529,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 446,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 22,
 // CHECK-NEXT:   "col": 6,
@@ -532,12 +537,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 441,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 452,
 // CHECK-NEXT:    "col": 12,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -554,7 +559,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 460,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 23,
 // CHECK-NEXT:   "col": 6,
@@ -562,12 +567,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 455,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 470,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -583,7 +588,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 478,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 24,
 // CHECK-NEXT:   "col": 6,
@@ -591,12 +596,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 473,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 496,
 // CHECK-NEXT:    "col": 24,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -611,18 +616,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 488,
 // CHECK-NEXT:     "col": 16,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 484,
 // CHECK-NEXT:      "col": 12,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 488,
 // CHECK-NEXT:      "col": 16,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -636,18 +641,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 495,
 // CHECK-NEXT:     "col": 23,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 491,
 // CHECK-NEXT:      "col": 19,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 495,
 // CHECK-NEXT:      "col": 23,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -664,7 +669,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 504,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 25,
 // CHECK-NEXT:   "col": 6,
@@ -672,12 +677,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 499,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 527,
 // CHECK-NEXT:    "col": 29,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -692,18 +697,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 514,
 // CHECK-NEXT:     "col": 16,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 510,
 // CHECK-NEXT:      "col": 12,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 514,
 // CHECK-NEXT:      "col": 16,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -717,18 +722,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 521,
 // CHECK-NEXT:     "col": 23,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 517,
 // CHECK-NEXT:      "col": 19,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 525,
 // CHECK-NEXT:      "col": 27,
 // CHECK-NEXT:      "tokLen": 2
 // CHECK-NEXT:     }
@@ -744,12 +749,12 @@ int main() {
 // CHECK-NEXT:      "kind": "IntegerLiteral",
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 525,
 // CHECK-NEXT:        "col": 27,
 // CHECK-NEXT:        "tokLen": 2
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 525,
 // CHECK-NEXT:        "col": 27,
 // CHECK-NEXT:        "tokLen": 2
 // CHECK-NEXT:       }
@@ -769,7 +774,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 545,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 26,
 // CHECK-NEXT:   "col": 16,
@@ -777,12 +782,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 530,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 9
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 555,
 // CHECK-NEXT:    "col": 26,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -799,7 +804,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 570,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 27,
 // CHECK-NEXT:   "col": 13,
@@ -807,12 +812,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 558,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 6
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 580,
 // CHECK-NEXT:    "col": 23,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -829,7 +834,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 595,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 28,
 // CHECK-NEXT:   "col": 13,
@@ -837,12 +842,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 583,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 6
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 605,
 // CHECK-NEXT:    "col": 23,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -859,7 +864,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 620,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 29,
 // CHECK-NEXT:   "col": 13,
@@ -867,12 +872,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 608,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 6
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 630,
 // CHECK-NEXT:    "col": 23,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -889,7 +894,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 638,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 30,
 // CHECK-NEXT:   "col": 6,
@@ -897,12 +902,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 633,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 650,
 // CHECK-NEXT:    "col": 18,
 // CHECK-NEXT:    "tokLen": 8
 // CHECK-NEXT:   }
@@ -918,7 +923,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 665,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 31,
 // CHECK-NEXT:   "col": 6,
@@ -926,12 +931,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 660,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 692,
 // CHECK-NEXT:    "col": 33,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -947,7 +952,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 700,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 32,
 // CHECK-NEXT:   "col": 6,
@@ -955,12 +960,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 695,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 723,
 // CHECK-NEXT:    "col": 29,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -976,7 +981,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionTemplateDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 751,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 35,
 // CHECK-NEXT:   "col": 3,
@@ -984,13 +989,13 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 727,
 // CHECK-NEXT:    "line": 34,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 8
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 760,
 // CHECK-NEXT:    "line": 35,
 // CHECK-NEXT:    "col": 12,
 // CHECK-NEXT:    "tokLen": 1
@@ -1002,19 +1007,19 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "TemplateTypeParmDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 746,
 // CHECK-NEXT:     "line": 34,
 // CHECK-NEXT:     "col": 20,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 737,
 // CHECK-NEXT:      "col": 11,
 // CHECK-NEXT:      "tokLen": 8
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 746,
 // CHECK-NEXT:      "col": 20,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -1029,19 +1034,19 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "FunctionDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 751,
 // CHECK-NEXT:     "line": 35,
 // CHECK-NEXT:     "col": 3,
 // CHECK-NEXT:     "tokLen": 6
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 749,
 // CHECK-NEXT:      "col": 1,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 760,
 // CHECK-NEXT:      "col": 12,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -1055,18 +1060,18 @@ int main() {
 // CHECK-NEXT:      "id": "0x{{.*}}",
 // CHECK-NEXT:      "kind": "ParmVarDecl",
 // CHECK-NEXT:      "loc": {
-// CHECK-NEXT:       "offset": {{[0-9]+}},
+// CHECK-NEXT:       "offset": 760,
 // CHECK-NEXT:       "col": 12,
 // CHECK-NEXT:       "tokLen": 1
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 758,
 // CHECK-NEXT:        "col": 10,
 // CHECK-NEXT:        "tokLen": 1
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 759,
 // CHECK-NEXT:        "col": 11,
 // CHECK-NEXT:        "tokLen": 1
 // CHECK-NEXT:       }
@@ -1084,7 +1089,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 769,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 37,
 // CHECK-NEXT:   "col": 6,
@@ -1092,12 +1097,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 764,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 782,
 // CHECK-NEXT:    "col": 19,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -1112,18 +1117,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 779,
 // CHECK-NEXT:     "col": 16,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 776,
 // CHECK-NEXT:      "col": 13,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 776,
 // CHECK-NEXT:      "col": 13,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     }
@@ -1137,12 +1142,12 @@ int main() {
 // CHECK-NEXT:    "kind": "CompoundStmt",
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 781,
 // CHECK-NEXT:      "col": 18,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 782,
 // CHECK-NEXT:      "col": 19,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -1155,7 +1160,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 789,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 38,
 // CHECK-NEXT:   "col": 6,
@@ -1163,12 +1168,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 784,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 807,
 // CHECK-NEXT:    "col": 24,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -1184,18 +1189,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 799,
 // CHECK-NEXT:     "col": 16,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 796,
 // CHECK-NEXT:      "col": 13,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 796,
 // CHECK-NEXT:      "col": 13,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     }
@@ -1209,12 +1214,12 @@ int main() {
 // CHECK-NEXT:    "kind": "CompoundStmt",
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 806,
 // CHECK-NEXT:      "col": 23,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 807,
 // CHECK-NEXT:      "col": 24,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -1222,3 +1227,189 @@ int main() {
 // CHECK-NEXT:   }
 // CHECK-NEXT:  ]
 // CHECK-NEXT: }
+
+
+// CHECK-NOT: {{^}}Dumping
+// CHECK:  "kind": "ClassTemplateDecl",
+// CHECK-NEXT:  "loc": {
+// CHECK-NEXT:   "offset": 905,
+// CHECK-NEXT:   "file": "{{.*}}",
+// CHECK-NEXT:   "line": 45,
+// CHECK-NEXT:   "col": 8,
+// CHECK-NEXT:   "tokLen": 11
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "range": {
+// CHECK-NEXT:   "begin": {
+// CHECK-NEXT:    "offset": 872,
+// CHECK-NEXT:    "line": 44,
+// CHECK-NEXT:    "col": 1,
+// CHECK-NEXT:    "tokLen": 8
+// CHECK-NEXT:   },
+// CHECK-NEXT:   "end": {
+// CHECK-NEXT:    "offset": 937,
+// CHECK-NEXT:    "line": 47,
+// CHECK-NEXT:    "col": 1,
+// CHECK-NEXT:    "tokLen": 1
+// CHECK-NEXT:   }
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "name": "TestFriends",
+// CHECK-NEXT:  "inner": [
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TemplateTypeParmDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 894,
+// CHECK-NEXT:     "line": 44,
+// CHECK-NEXT:     "col": 23,
+// CHECK-NEXT:     "tokLen": 2
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 882,
+// CHECK-NEXT:      "col": 11,
+// CHECK-NEXT:      "tokLen": 8
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 894,
+// CHECK-NEXT:      "col": 23,
+// CHECK-NEXT:      "tokLen": 2
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isReferenced": true,
+// CHECK-NEXT:    "name": "Ts",
+// CHECK-NEXT:    "tagUsed": "typename",
+// CHECK-NEXT:    "depth": 0,
+// CHECK-NEXT:    "index": 0,
+// CHECK-NEXT:    "isParameterPack": true
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "CXXRecordDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 905,
+// CHECK-NEXT:     "line": 45,
+// CHECK-NEXT:     "col": 8,
+// CHECK-NEXT:     "tokLen": 11
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 898,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 6
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 937,
+// CHECK-NEXT:      "line": 47,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "name": "TestFriends",
+// CHECK-NEXT:    "tagUsed": "struct",
+// CHECK-NEXT:    "completeDefinition": true,
+// CHECK-NEXT:    "definitionData": {
+// CHECK-NEXT:     "canConstDefaultInit": true,
+// CHECK-NEXT:     "copyAssign": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "copyCtor": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "defaultCtor": {
+// CHECK-NEXT:      "defaultedIsConstexpr": true,
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "isConstexpr": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "dtor": {
+// CHECK-NEXT:      "irrelevant": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "hasConstexprNonCopyMoveConstructor": true,
+// CHECK-NEXT:     "isAggregate": true,
+// CHECK-NEXT:     "isEmpty": true,
+// CHECK-NEXT:     "isLiteral": true,
+// CHECK-NEXT:     "isPOD": true,
+// CHECK-NEXT:     "isStandardLayout": true,
+// CHECK-NEXT:     "isTrivial": true,
+// CHECK-NEXT:     "isTriviallyCopyable": true,
+// CHECK-NEXT:     "moveAssign": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "moveCtor": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXRecordDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 905,
+// CHECK-NEXT:       "line": 45,
+// CHECK-NEXT:       "col": 8,
+// CHECK-NEXT:       "tokLen": 11
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 898,
+// CHECK-NEXT:        "col": 1,
+// CHECK-NEXT:        "tokLen": 6
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 905,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 11
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isImplicit": true,
+// CHECK-NEXT:      "name": "TestFriends",
+// CHECK-NEXT:      "tagUsed": "struct"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "FriendDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 930,
+// CHECK-NEXT:       "line": 46,
+// CHECK-NEXT:       "col": 12,
+// CHECK-NEXT:       "tokLen": 2
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 923,
+// CHECK-NEXT:        "col": 5,
+// CHECK-NEXT:        "tokLen": 6
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 932,
+// CHECK-NEXT:        "col": 14,
+// CHECK-NEXT:        "tokLen": 3
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "Ts"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isPackExpansion": true
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   }
+// CHECK-NEXT:  ]
+// CHECK-NEXT: }
diff --git a/clang/test/AST/cxx2c-variadic-friends.cpp b/clang/test/AST/cxx2c-variadic-friends.cpp
new file mode 100644
index 000000000000000..fc84e7346fe0364
--- /dev/null
+++ b/clang/test/AST/cxx2c-variadic-friends.cpp
@@ -0,0 +1,81 @@
+// RUN: %clang_cc1 -fsyntax-only -ast-dump -std=c++2c %s | FileCheck %s
+// RUN: %clang_cc1 -ast-print -std=c++2c %s | FileCheck %s --check-prefix=PRINT
+// RUN: %clang_cc1 -emit-pch -std=c++2c -o %t %s
+// RUN: %clang_cc1 -x c++ -std=c++2c -include-pch %t -ast-dump-all /dev/null
+
+struct S;
+template <typename> struct TS; // #template
+
+// CHECK-LABEL: CXXRecordDecl {{.*}} struct Friends
+// PRINT-LABEL: struct Friends {
+struct Friends {
+  // CHECK: FriendDecl {{.*}} 'int'
+  // CHECK-NEXT: FriendDecl {{.*}} 'long'
+  // PRINT-NEXT: friend int;
+  // PRINT-NEXT: friend long;
+  friend int, long;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'int'
+  // CHECK-NEXT: FriendDecl {{.*}} 'long'
+  // CHECK-NEXT: FriendDecl {{.*}} 'char'
+  // PRINT-NEXT: friend int;
+  // PRINT-NEXT: friend long;
+  // PRINT-NEXT: friend char;
+  friend int, long, char;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'S'
+  // PRINT-NEXT: friend S;
+  friend S;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'S'
+  // CHECK-NEXT: FriendDecl {{.*}} 'S'
+  // CHECK-NEXT: FriendDecl {{.*}} 'S'
+  // PRINT-NEXT: friend S;
+  // PRINT-NEXT: friend S;
+  // PRINT-NEXT: friend S;
+  friend S, S, S;
+
+  // CHECK-NEXT: FriendDecl
+  // CHECK-NEXT: ClassTemplateDecl {{.*}} friend TS
+  // PRINT-NEXT: friend template <typename> struct TS;
+  template <typename> friend struct TS;
+};
+
+namespace specialisations {
+template<class T>
+struct C {
+  template<class U> struct Nested;
+};
+
+struct N {
+  template<class U> class C;
+};
+
+// CHECK-LABEL: ClassTemplateDecl {{.*}} Variadic
+// PRINT-LABEL: template <typename ...Pack> struct Variadic {
+template <typename ...Pack> struct Variadic {
+  // CHECK: FriendDecl {{.*}} 'Pack'...
+  // CHECK-NEXT: FriendDecl {{.*}} 'long'
+  // CHECK-NEXT: FriendDecl {{.*}} 'Pack'...
+  // PRINT-NEXT: friend Pack...;
+  // PRINT-NEXT: friend long;
+  // PRINT-NEXT: friend Pack...;
+  friend Pack..., long, Pack...;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'TS<Pack>'...
+  // PRINT-NEXT: friend TS<Pack>...;
+  friend TS<Pack>...;
+};
+
+// CHECK-LABEL: ClassTemplateDecl {{.*}} S2
+// PRINT-LABEL: template <class ...Ts> struct S2 {
+template<class ...Ts> struct S2 {
+  // CHECK: FriendDecl {{.*}} 'class C<Ts>':'C<Ts>'...
+  // PRINT-NEXT: friend class C<Ts>...;
+  friend class C<Ts>...;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'class N::C<Ts>':'C<Ts>'...
+  // PRINT-NEXT: friend class N::C<Ts>...
+  friend class N::C<Ts>...;
+};
+}
diff --git a/clang/test/Analysis/asm.cpp b/clang/test/Analysis/asm.cpp
index 1180063502168f8..3181aea870c8aad 100644
--- a/clang/test/Analysis/asm.cpp
+++ b/clang/test/Analysis/asm.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker debug.ExprInspection -fheinous-gnu-extensions -w %s -verify
+// RUN: %clang_analyze_cc1 -triple=x86_64-unknown-unknown \
+// RUN:      -analyzer-checker debug.ExprInspection,core -fheinous-gnu-extensions -w %s -verify
 
 int clang_analyzer_eval(int);
 
@@ -10,3 +11,32 @@ void testRValueOutput() {
   clang_analyzer_eval(global == 1); // expected-warning{{UNKNOWN}}
   clang_analyzer_eval(ref == 1);    // expected-warning{{UNKNOWN}}
 }
+
+void *MyMemcpy(void *d, const void *s, const int n) {
+  asm volatile (
+    "cld\n rep movsb\n"
+    :: "S" (s), "D" (d), "c" (n) : "memory"
+  );
+  return d;
+}
+
+void testInlineAsmMemcpy(void)
+{
+    int a, b = 10, c;
+    MyMemcpy(&a, &b, sizeof(b));
+    c = a; // no-warning
+}
+
+void testInlineAsmMemcpyArray(void)
+{
+    int a[10], b[10] = {}, c;
+    MyMemcpy(&a, &b, sizeof(b));
+    c = a[8]; // no-warning
+}
+
+void testInlineAsmMemcpyUninit(void)
+{
+    int a[10], b[10] = {}, c;
+    MyMemcpy(&a[1], &b[1], sizeof(b) - sizeof(b[1]));
+    c = a[0]; // expected-warning{{Assigned value is garbage or undefined}}
+}
diff --git a/clang/test/Analysis/builtin_signbit.cpp b/clang/test/Analysis/builtin_signbit.cpp
index be10f0950f69b7c..ad185d855cfff30 100644
--- a/clang/test/Analysis/builtin_signbit.cpp
+++ b/clang/test/Analysis/builtin_signbit.cpp
@@ -12,103 +12,83 @@ long double ld = -1.0L;
 // CHECK-BE32-LABEL: define dso_local void @_Z12test_signbitv(
 // CHECK-BE32-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-BE32-NEXT:  entry:
-// CHECK-BE32-NEXT:    [[TMP0:%.*]] = lshr i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128), 64
-// CHECK-BE32-NEXT:    [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
-// CHECK-BE32-NEXT:    [[TMP2:%.*]] = icmp slt i64 [[TMP1]], 0
-// CHECK-BE32-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP2]] to i8
+// CHECK-BE32-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-BE32-NEXT:    [[TMP0:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-BE32-NEXT:    [[TMP1:%.*]] = bitcast ppc_fp128 [[TMP0]] to i128
+// CHECK-BE32-NEXT:    [[TMP2:%.*]] = lshr i128 [[TMP1]], 64
+// CHECK-BE32-NEXT:    [[TMP3:%.*]] = trunc i128 [[TMP2]] to i64
+// CHECK-BE32-NEXT:    [[TMP4:%.*]] = icmp slt i64 [[TMP3]], 0
+// CHECK-BE32-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP4]] to i8
 // CHECK-BE32-NEXT:    store i8 [[FROMBOOL]], ptr @b, align 1
-// CHECK-BE32-NEXT:    [[TMP3:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-BE32-NEXT:    [[TMP4:%.*]] = bitcast ppc_fp128 [[TMP3]] to i128
-// CHECK-BE32-NEXT:    [[TMP5:%.*]] = lshr i128 [[TMP4]], 64
-// CHECK-BE32-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
-// CHECK-BE32-NEXT:    [[TMP7:%.*]] = icmp slt i64 [[TMP6]], 0
+// CHECK-BE32-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-BE32-NEXT:    [[TMP5:%.*]] = load double, ptr @d, align 8
+// CHECK-BE32-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP5]] to float
+// CHECK-BE32-NEXT:    [[TMP6:%.*]] = bitcast float [[CONV]] to i32
+// CHECK-BE32-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
 // CHECK-BE32-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[TMP7]] to i8
 // CHECK-BE32-NEXT:    store i8 [[FROMBOOL1]], ptr @b, align 1
 // CHECK-BE32-NEXT:    store i8 0, ptr @b, align 1
-// CHECK-BE32-NEXT:    [[TMP8:%.*]] = load double, ptr @d, align 8
-// CHECK-BE32-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP8]] to float
-// CHECK-BE32-NEXT:    [[TMP9:%.*]] = bitcast float [[CONV]] to i32
-// CHECK-BE32-NEXT:    [[TMP10:%.*]] = icmp slt i32 [[TMP9]], 0
-// CHECK-BE32-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP10]] to i8
+// CHECK-BE32-NEXT:    [[TMP8:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-BE32-NEXT:    [[TMP9:%.*]] = bitcast ppc_fp128 [[TMP8]] to i128
+// CHECK-BE32-NEXT:    [[TMP10:%.*]] = lshr i128 [[TMP9]], 64
+// CHECK-BE32-NEXT:    [[TMP11:%.*]] = trunc i128 [[TMP10]] to i64
+// CHECK-BE32-NEXT:    [[TMP12:%.*]] = icmp slt i64 [[TMP11]], 0
+// CHECK-BE32-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP12]] to i8
 // CHECK-BE32-NEXT:    store i8 [[FROMBOOL2]], ptr @b, align 1
-// CHECK-BE32-NEXT:    [[TMP11:%.*]] = lshr i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128), 64
-// CHECK-BE32-NEXT:    [[TMP12:%.*]] = trunc i128 [[TMP11]] to i64
-// CHECK-BE32-NEXT:    [[TMP13:%.*]] = icmp slt i64 [[TMP12]], 0
-// CHECK-BE32-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[TMP13]] to i8
-// CHECK-BE32-NEXT:    store i8 [[FROMBOOL3]], ptr @b, align 1
-// CHECK-BE32-NEXT:    [[TMP14:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-BE32-NEXT:    [[TMP15:%.*]] = bitcast ppc_fp128 [[TMP14]] to i128
-// CHECK-BE32-NEXT:    [[TMP16:%.*]] = lshr i128 [[TMP15]], 64
-// CHECK-BE32-NEXT:    [[TMP17:%.*]] = trunc i128 [[TMP16]] to i64
-// CHECK-BE32-NEXT:    [[TMP18:%.*]] = icmp slt i64 [[TMP17]], 0
-// CHECK-BE32-NEXT:    [[FROMBOOL4:%.*]] = zext i1 [[TMP18]] to i8
-// CHECK-BE32-NEXT:    store i8 [[FROMBOOL4]], ptr @b, align 1
 // CHECK-BE32-NEXT:    ret void
 //
 // CHECK-BE64-LABEL: define dso_local void @_Z12test_signbitv(
 // CHECK-BE64-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-BE64-NEXT:  entry:
-// CHECK-BE64-NEXT:    [[TMP0:%.*]] = lshr i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128), 64
-// CHECK-BE64-NEXT:    [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
-// CHECK-BE64-NEXT:    [[TMP2:%.*]] = icmp slt i64 [[TMP1]], 0
-// CHECK-BE64-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP2]] to i8
+// CHECK-BE64-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-BE64-NEXT:    [[TMP0:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-BE64-NEXT:    [[TMP1:%.*]] = bitcast ppc_fp128 [[TMP0]] to i128
+// CHECK-BE64-NEXT:    [[TMP2:%.*]] = lshr i128 [[TMP1]], 64
+// CHECK-BE64-NEXT:    [[TMP3:%.*]] = trunc i128 [[TMP2]] to i64
+// CHECK-BE64-NEXT:    [[TMP4:%.*]] = icmp slt i64 [[TMP3]], 0
+// CHECK-BE64-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP4]] to i8
 // CHECK-BE64-NEXT:    store i8 [[FROMBOOL]], ptr @b, align 1
-// CHECK-BE64-NEXT:    [[TMP3:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-BE64-NEXT:    [[TMP4:%.*]] = bitcast ppc_fp128 [[TMP3]] to i128
-// CHECK-BE64-NEXT:    [[TMP5:%.*]] = lshr i128 [[TMP4]], 64
-// CHECK-BE64-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
-// CHECK-BE64-NEXT:    [[TMP7:%.*]] = icmp slt i64 [[TMP6]], 0
+// CHECK-BE64-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-BE64-NEXT:    [[TMP5:%.*]] = load double, ptr @d, align 8
+// CHECK-BE64-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP5]] to float
+// CHECK-BE64-NEXT:    [[TMP6:%.*]] = bitcast float [[CONV]] to i32
+// CHECK-BE64-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
 // CHECK-BE64-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[TMP7]] to i8
 // CHECK-BE64-NEXT:    store i8 [[FROMBOOL1]], ptr @b, align 1
 // CHECK-BE64-NEXT:    store i8 0, ptr @b, align 1
-// CHECK-BE64-NEXT:    [[TMP8:%.*]] = load double, ptr @d, align 8
-// CHECK-BE64-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP8]] to float
-// CHECK-BE64-NEXT:    [[TMP9:%.*]] = bitcast float [[CONV]] to i32
-// CHECK-BE64-NEXT:    [[TMP10:%.*]] = icmp slt i32 [[TMP9]], 0
-// CHECK-BE64-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP10]] to i8
+// CHECK-BE64-NEXT:    [[TMP8:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-BE64-NEXT:    [[TMP9:%.*]] = bitcast ppc_fp128 [[TMP8]] to i128
+// CHECK-BE64-NEXT:    [[TMP10:%.*]] = lshr i128 [[TMP9]], 64
+// CHECK-BE64-NEXT:    [[TMP11:%.*]] = trunc i128 [[TMP10]] to i64
+// CHECK-BE64-NEXT:    [[TMP12:%.*]] = icmp slt i64 [[TMP11]], 0
+// CHECK-BE64-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP12]] to i8
 // CHECK-BE64-NEXT:    store i8 [[FROMBOOL2]], ptr @b, align 1
-// CHECK-BE64-NEXT:    [[TMP11:%.*]] = lshr i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128), 64
-// CHECK-BE64-NEXT:    [[TMP12:%.*]] = trunc i128 [[TMP11]] to i64
-// CHECK-BE64-NEXT:    [[TMP13:%.*]] = icmp slt i64 [[TMP12]], 0
-// CHECK-BE64-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[TMP13]] to i8
-// CHECK-BE64-NEXT:    store i8 [[FROMBOOL3]], ptr @b, align 1
-// CHECK-BE64-NEXT:    [[TMP14:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-BE64-NEXT:    [[TMP15:%.*]] = bitcast ppc_fp128 [[TMP14]] to i128
-// CHECK-BE64-NEXT:    [[TMP16:%.*]] = lshr i128 [[TMP15]], 64
-// CHECK-BE64-NEXT:    [[TMP17:%.*]] = trunc i128 [[TMP16]] to i64
-// CHECK-BE64-NEXT:    [[TMP18:%.*]] = icmp slt i64 [[TMP17]], 0
-// CHECK-BE64-NEXT:    [[FROMBOOL4:%.*]] = zext i1 [[TMP18]] to i8
-// CHECK-BE64-NEXT:    store i8 [[FROMBOOL4]], ptr @b, align 1
 // CHECK-BE64-NEXT:    ret void
 //
 // CHECK-LE-LABEL: define dso_local void @_Z12test_signbitv(
 // CHECK-LE-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-LE-NEXT:  entry:
-// CHECK-LE-NEXT:    [[TMP0:%.*]] = icmp slt i64 trunc (i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128) to i64), 0
-// CHECK-LE-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP0]] to i8
+// CHECK-LE-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast ppc_fp128 [[TMP0]] to i128
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = trunc i128 [[TMP1]] to i64
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp slt i64 [[TMP2]], 0
+// CHECK-LE-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP3]] to i8
 // CHECK-LE-NEXT:    store i8 [[FROMBOOL]], ptr @b, align 1
-// CHECK-LE-NEXT:    [[TMP1:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-LE-NEXT:    [[TMP2:%.*]] = bitcast ppc_fp128 [[TMP1]] to i128
-// CHECK-LE-NEXT:    [[TMP3:%.*]] = trunc i128 [[TMP2]] to i64
-// CHECK-LE-NEXT:    [[TMP4:%.*]] = icmp slt i64 [[TMP3]], 0
-// CHECK-LE-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[TMP4]] to i8
+// CHECK-LE-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-LE-NEXT:    [[TMP4:%.*]] = load double, ptr @d, align 8
+// CHECK-LE-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP4]] to float
+// CHECK-LE-NEXT:    [[TMP5:%.*]] = bitcast float [[CONV]] to i32
+// CHECK-LE-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP5]], 0
+// CHECK-LE-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[TMP6]] to i8
 // CHECK-LE-NEXT:    store i8 [[FROMBOOL1]], ptr @b, align 1
 // CHECK-LE-NEXT:    store i8 0, ptr @b, align 1
-// CHECK-LE-NEXT:    [[TMP5:%.*]] = load double, ptr @d, align 8
-// CHECK-LE-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP5]] to float
-// CHECK-LE-NEXT:    [[TMP6:%.*]] = bitcast float [[CONV]] to i32
-// CHECK-LE-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
-// CHECK-LE-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP7]] to i8
+// CHECK-LE-NEXT:    [[TMP7:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-LE-NEXT:    [[TMP8:%.*]] = bitcast ppc_fp128 [[TMP7]] to i128
+// CHECK-LE-NEXT:    [[TMP9:%.*]] = trunc i128 [[TMP8]] to i64
+// CHECK-LE-NEXT:    [[TMP10:%.*]] = icmp slt i64 [[TMP9]], 0
+// CHECK-LE-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP10]] to i8
 // CHECK-LE-NEXT:    store i8 [[FROMBOOL2]], ptr @b, align 1
-// CHECK-LE-NEXT:    [[TMP8:%.*]] = icmp slt i64 trunc (i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128) to i64), 0
-// CHECK-LE-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[TMP8]] to i8
-// CHECK-LE-NEXT:    store i8 [[FROMBOOL3]], ptr @b, align 1
-// CHECK-LE-NEXT:    [[TMP9:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-LE-NEXT:    [[TMP10:%.*]] = bitcast ppc_fp128 [[TMP9]] to i128
-// CHECK-LE-NEXT:    [[TMP11:%.*]] = trunc i128 [[TMP10]] to i64
-// CHECK-LE-NEXT:    [[TMP12:%.*]] = icmp slt i64 [[TMP11]], 0
-// CHECK-LE-NEXT:    [[FROMBOOL4:%.*]] = zext i1 [[TMP12]] to i8
-// CHECK-LE-NEXT:    store i8 [[FROMBOOL4]], ptr @b, align 1
 // CHECK-LE-NEXT:    ret void
 //
 void test_signbit()
diff --git a/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp b/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
index d23d00ccdeebc1b..497a9328c115618 100644
--- a/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
+++ b/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
@@ -11,6 +11,8 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -pedantic %s -DTEST11
 // RUN: %clang_cc1 -fsyntax-only -verify -pedantic %s -DTEST12
 // RUN: %clang_cc1 -fsyntax-only -verify -pedantic %s -DTEST13
+// RUN: %clang_cc1 -fsyntax-only -verify -pedantic %s -DTEST14
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm-only -verify -pedantic %s -DTEST15
 
 #if TEST1
 int main; // expected-error{{main cannot be declared as a variable in the global scope}}
@@ -78,12 +80,14 @@ namespace ns {
 extern "C" struct A { int main(); }; // ok
 
 namespace c {
-  extern "C" void main(); // expected-warning {{'main' should not be 'extern "C"'}}
+  extern "C" void main(); // expected-error {{'main' must return 'int'}} \
+                          // expected-warning {{'main' should not be 'extern "C"'}}
 }
 
 extern "C" {
   namespace Z {
-    void main(); // expected-warning {{'main' should not be 'extern "C"'}}
+    void main(); // expected-error {{'main' must return 'int'}} \
+                 // expected-warning {{'main' should not be 'extern "C"'}}
   }
 }
 
@@ -102,11 +106,6 @@ extern "C++" {
   int main(); // expected-warning {{'main' should not be 'extern "C++"'}}
 }
 
-extern "C" {
-  int main(); // expected-warning {{'main' should not be 'extern "C"'}}
-}
-
-extern "C" int main(); // expected-warning {{'main' should not be 'extern "C"'}}
 extern "C++" int main(); // expected-warning {{'main' should not be 'extern "C++"'}}
 
 namespace ns1 {
@@ -122,6 +121,21 @@ namespace ns2 {
   extern "C++" void main() {} // ok
 }
 
+#elif TEST14
+extern "C" {
+  int main(); // expected-warning {{'main' should not be 'extern "C"'}}
+}
+
+extern "C" int main(); // expected-warning {{'main' should not be 'extern "C"'}}
+
+#elif TEST15
+extern "C" __attribute__((visibility("default"))) __attribute__((weak))
+int main(); // expected-warning {{'main' should not be 'extern "C"'}}
+
+unsigned long g() {
+  return reinterpret_cast<unsigned long>(&main); // expected-warning {{referring to 'main' within an expression is a Clang extension}}
+}
+
 #else
 #error Unknown Test
 #endif
diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p1.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p1.cpp
index ddf82f432c2eab0..a195e0548152d6c 100644
--- a/clang/test/CXX/class/class.compare/class.compare.default/p1.cpp
+++ b/clang/test/CXX/class/class.compare/class.compare.default/p1.cpp
@@ -16,6 +16,8 @@ struct A {
   bool operator<(const A&) const;
   bool operator<=(const A&) const = default;
   bool operator==(const A&) const && = default; // expected-error {{ref-qualifier '&&' is not allowed on a defaulted comparison operator}}
+  bool operator<=(const A&&) const = default; // expected-error {{invalid parameter type for defaulted relational comparison operator; found 'const A &&', expected 'const A &'}}
+  bool operator<=(const int&) const = default; // expected-error {{invalid parameter type for defaulted relational comparison operator; found 'const int &', expected 'const A &'}}
   bool operator>=(const A&) const volatile = default; // expected-error {{defaulted comparison function must not be volatile}}
   bool operator<=>(const A&) = default; // expected-error {{defaulted member three-way comparison operator must be const-qualified}}
   bool operator>=(const B&) const = default; // expected-error-re {{invalid parameter type for defaulted relational comparison operator; found 'const B &', expected 'const A &'{{$}}}}
diff --git a/clang/test/CXX/drs/cwg20xx.cpp b/clang/test/CXX/drs/cwg20xx.cpp
index cd2f26360d40215..b2dc5a9865e382c 100644
--- a/clang/test/CXX/drs/cwg20xx.cpp
+++ b/clang/test/CXX/drs/cwg20xx.cpp
@@ -401,11 +401,34 @@ namespace cwg2083 { // cwg2083: partial
 #endif
 }
 
-namespace cwg2091 { // cwg2091: 2.7
+namespace cwg2091 { // cwg2091: 10
 template<int &> struct X;
 template<int &N> void f(X<N>&);
 int n;
 void g(X<n> &x) { f(x); }
+
+namespace GH42233 {
+enum E { I };
+
+class AA { };
+E EV[1] = {I};
+
+template<class ENUM, const ENUM* const VALUES>
+struct S
+{
+  template< class E, const E* const V>
+    friend AA& operator<<( AA& os, const S<E,V>& e );
+};
+
+int f()
+{
+  S< E, EV > x;
+
+  AA a;
+  a << x;
+  return 0;
+}
+} // namespace GH42233
 } // namespace cwg2091 
 
 namespace cwg2094 { // cwg2094: 5
diff --git a/clang/test/CXX/drs/cwg25xx.cpp b/clang/test/CXX/drs/cwg25xx.cpp
index 0d9f5eac23866a2..1924008f15ba580 100644
--- a/clang/test/CXX/drs/cwg25xx.cpp
+++ b/clang/test/CXX/drs/cwg25xx.cpp
@@ -92,6 +92,30 @@ using ::cwg2521::operator""_div;
 #endif
 } // namespace cwg2521
 
+namespace cwg2547 { // cwg2547: 20
+#if __cplusplus >= 202302L
+struct S;
+// since-cxx23-note@-1 {{forward declaration of 'cwg2547::S'}}
+// since-cxx23-note@-2 {{forward declaration of 'cwg2547::S'}}
+// since-cxx23-note@-3 {{forward declaration of 'cwg2547::S'}}
+bool operator==(S, S) = default;  // error: S is not complete
+// since-cxx23-error@-1 {{variable has incomplete type 'S'}}
+// since-cxx23-error@-2 {{variable has incomplete type 'S'}}
+// since-cxx23-error@-3 {{equality comparison operator is not a friend of incomplete class 'cwg2547::S'}}
+struct S {
+  friend bool operator==(S, const S&) = default; // error: parameters of different types
+  // since-cxx23-error@-1 {{parameters for defaulted equality comparison operator must have the same type (found 'S' vs 'const S &')}}
+};
+enum E { };
+bool operator==(E, E) = default;  // error: not a member or friend of a class
+// since-cxx23-error@-1 {{invalid parameter type for non-member defaulted equality comparison operator; found 'E', expected class or reference to a constant class}}
+
+struct S2 {
+  bool operator==(this int, S2) = default;
+  // since-cxx23-error@-1 {{invalid parameter type for defaulted equality comparison operator; found 'int', expected 'const cwg2547::S2 &'}}
+};
+#endif
+} // namespace cwg2547
 
 #if __cplusplus >= 202302L
 namespace cwg2553 { // cwg2553: 18 review 2023-07-14
@@ -253,6 +277,41 @@ static_assert(__is_layout_compatible(U, V), "");
 #endif
 } // namespace cwg2583
 
+namespace cwg2586 { // cwg2586: 20
+#if __cplusplus >= 202302L
+struct X {
+  X& operator=(this X&, const X&) = default;
+  X& operator=(this X&, X&) = default;
+  X& operator=(this X&&, X&&) = default;
+  // FIXME: The notes could be clearer on *how* the type differs
+  // e.g., "if an explicit object parameter is used it must be of type reference to 'X'"
+  X& operator=(this int, const X&) = default;
+  // since-cxx23-warning@-1 {{explicitly defaulted copy assignment operator is implicitly deleted}}
+  // since-cxx23-note@-2 {{function is implicitly deleted because its declared type does not match the type of an implicit copy assignment operator}}
+  X& operator=(this X, const X&) = default;
+  // since-cxx23-warning@-1 {{explicitly defaulted copy assignment operator is implicitly deleted}}
+  // since-cxx23-note@-2 {{function is implicitly deleted because its declared type does not match the type of an implicit copy assignment operator}}
+};
+struct Y {
+  void operator=(this int, const Y&); // This is copy constructor, suppresses implicit declaration
+};
+static_assert([]<typename T = Y>{
+  return !requires(T t, const T& ct) { t = ct; };
+}());
+
+struct Z {
+  bool operator==(this const Z&, const Z&) = default;
+  bool operator==(this Z, Z) = default;
+  bool operator==(this Z, const Z&) = default;
+  // since-cxx23-error@-1 {{parameters for defaulted equality comparison operator must have the same type (found 'Z' vs 'const Z &')}}
+  bool operator==(this const Z&, Z) = default;
+  // since-cxx23-error@-1 {{parameters for defaulted equality comparison operator must have the same type (found 'const Z &' vs 'Z')}}
+  bool operator==(this int, Z) = default;
+  // since-cxx23-error@-1 {{invalid parameter type for defaulted equality comparison operator; found 'int', expected 'const cwg2586::Z &'}}
+};
+#endif
+} // namespace cwg2586
+
 namespace cwg2598 { // cwg2598: 18
 #if __cplusplus >= 201103L
 struct NonLiteral {
diff --git a/clang/test/CXX/drs/cwg29xx.cpp b/clang/test/CXX/drs/cwg29xx.cpp
new file mode 100644
index 000000000000000..8cac9f283980b6b
--- /dev/null
+++ b/clang/test/CXX/drs/cwg29xx.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -std=c++98 -pedantic-errors -verify=expected,cxx98 %s
+// RUN: %clang_cc1 -std=c++11 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++14 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++17 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++20 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++23 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++2c -pedantic-errors -verify=expected %s
+
+namespace cwg2917 { // cwg2917: 20 open 2024-07-30
+template <typename>
+class Foo;
+
+template<class ...> // cxx98-error {{variadic templates are a C++11 extension}}
+struct C {
+  struct Nested { };
+};
+
+struct S {
+  template <typename>
+  friend class Foo, int; // expected-error {{a friend declaration that befriends a template must contain exactly one type-specifier}}
+
+  template <typename ...Ts> // cxx98-error {{variadic templates are a C++11 extension}}
+  friend class C<Ts>::Nested...; // expected-error {{friend declaration expands pack 'Ts' that is declared it its own template parameter list}}
+};
+} // namespace cwg2917
diff --git a/clang/test/CodeGen/PowerPC/transparent_union.c b/clang/test/CodeGen/PowerPC/transparent_union.c
new file mode 100644
index 000000000000000..968a385c0ee45fe
--- /dev/null
+++ b/clang/test/CodeGen/PowerPC/transparent_union.c
@@ -0,0 +1,67 @@
+// RUN: %clang_cc1 -triple powerpc64le-unknown-linux -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-64
+// RUN: %clang_cc1 -triple powerpc64-unknown-linux -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-64
+// RUN: %clang_cc1 -triple powerpc-unknown-linux -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-32
+// RUN: %clang_cc1 -triple powerpc64-unknown-aix -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-64
+// RUN: %clang_cc1 -triple powerpc-unknown-aix -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-32
+
+typedef union tu_c {
+  signed char a;
+  signed char b;
+} tu_c_t __attribute__((transparent_union));
+
+typedef union tu_s {
+  short a;
+} tu_s_t __attribute__((transparent_union));
+
+typedef union tu_us {
+  unsigned short a;
+} tu_us_t __attribute__((transparent_union));
+
+typedef union tu_l {
+  long a;
+} tu_l_t __attribute__((transparent_union));
+
+// CHECK-LABEL: define{{.*}} void @ftest0(
+// CHECK-SAME: i8 noundef signext [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+void ftest0(tu_c_t uc) { }
+
+// CHECK-LABEL: define{{.*}} void @ftest1(
+// CHECK-SAME: i16 noundef signext [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+void ftest1(tu_s_t uc) { }
+
+// CHECK-LABEL: define{{.*}} void @ftest2(
+// CHECK-SAME: i16 noundef zeroext [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+void ftest2(tu_us_t uc) { }
+
+// CHECK-64-LABEL: define{{.*}} void @ftest3(
+// CHECK-64-SAME: i64 [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-64-NEXT:  [[ENTRY:.*:]]
+// CHECK-64-NEXT:    ret void
+//
+// CHECK-32-LABEL: define{{.*}} void @ftest3(
+// CHECK-32-SAME: i32 [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-32-NEXT:  [[ENTRY:.*:]]
+// CHECK-32-NEXT:    ret void
+void ftest3(tu_l_t uc) { }
+
+typedef union etest {
+  enum flag {red, yellow, blue} fl;
+  enum weekend {sun, sat} b;
+} etest_t __attribute__((transparent_union));
+
+// CHECK-LABEL: define{{.*}} void @ftest4(
+// CHECK-SAME: i8 noundef zeroext [[A_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+void ftest4(etest_t a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv-sdata-module-flag.c b/clang/test/CodeGen/RISCV/riscv-sdata-module-flag.c
index 9bd69e7995877e4..83beabe85b1f4dd 100644
--- a/clang/test/CodeGen/RISCV/riscv-sdata-module-flag.c
+++ b/clang/test/CodeGen/RISCV/riscv-sdata-module-flag.c
@@ -21,23 +21,19 @@
 // RUN:   | FileCheck %s -check-prefix=RV64-S2G4
 // RUN: %clang --target=riscv64-unknown-elf %s -S -emit-llvm -msmall-data-threshold=16 -o - \
 // RUN:   | FileCheck %s -check-prefix=RV64-T16
-// RUN: %clang --target=riscv64-linux-android %s -S -emit-llvm -o - \
-// RUN:   | FileCheck %s -check-prefix=RV64-ANDROID
-// RUN: %clang --target=riscv64-linux-android %s -S -emit-llvm -msmall-data-limit=8 -o - \
-// RUN:   | FileCheck %s -check-prefix=RV64-ANDROID
 // RUN: %clang --target=riscv64-unknown-elf %s -S -emit-llvm -fpic -o - \
 // RUN:   | FileCheck %s -check-prefix=RV64-PIC
 
 void test(void) {}
 
-// RV32-DEFAULT: !{i32 8, !"SmallDataLimit", i32 8}
+// RV32-DEFAULT: !{i32 8, !"SmallDataLimit", i32 0}
 // RV32-G4:      !{i32 8, !"SmallDataLimit", i32 4}
 // RV32-S0:      !{i32 8, !"SmallDataLimit", i32 0}
 // RV32-S2G4:    !{i32 8, !"SmallDataLimit", i32 4}
 // RV32-T16:     !{i32 8, !"SmallDataLimit", i32 16}
 // RV32-PIC:     !{i32 8, !"SmallDataLimit", i32 0}
 
-// RV64-DEFAULT: !{i32 8, !"SmallDataLimit", i32 8}
+// RV64-DEFAULT: !{i32 8, !"SmallDataLimit", i32 0}
 // RV64-G4:      !{i32 8, !"SmallDataLimit", i32 4}
 // RV64-S0:      !{i32 8, !"SmallDataLimit", i32 0}
 // RV64-S2G4:    !{i32 8, !"SmallDataLimit", i32 4}
@@ -48,6 +44,3 @@ void test(void) {}
 // The value will be passed by module flag instead of target feature.
 // RV32-S0-NOT: +small-data-limit=
 // RV64-S0-NOT: +small-data-limit=
-
-// RV64-ANDROID-NOT: small-data-limit
-// RV64-ANDROID:   !{i32 8, !"SmallDataLimit", i32 0}
diff --git a/clang/test/CodeGen/aarch64-elf-pauthabi.c b/clang/test/CodeGen/aarch64-elf-pauthabi.c
index d60dcee3618887e..023fa8c18e1300b 100644
--- a/clang/test/CodeGen/aarch64-elf-pauthabi.c
+++ b/clang/test/CodeGen/aarch64-elf-pauthabi.c
@@ -1,3 +1,5 @@
+//// TODO: also test with -fptrauth-elf-got when the driver flag is supported
+
 // RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
 // RUN:   -fptrauth-intrinsics \
 // RUN:   -fptrauth-calls \
@@ -5,8 +7,11 @@
 // RUN:   -fptrauth-auth-traps \
 // RUN:   -fptrauth-vtable-pointer-address-discrimination \
 // RUN:   -fptrauth-vtable-pointer-type-discrimination \
-// RUN:   -fptrauth-init-fini %s \
-// RUN:   -fptrauth-init-fini-address-discrimination %s | \
+// RUN:   -fptrauth-init-fini \
+// RUN:   -fptrauth-init-fini-address-discrimination \
+// RUN:   -fptrauth-indirect-gotos \
+// RUN:   -fptrauth-type-info-vtable-pointer-discrimination \
+// RUN:   -fptrauth-function-pointer-type-discrimination %s | \
 // RUN:   FileCheck %s --check-prefix=ALL
 
 // RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
@@ -37,8 +42,19 @@
 // RUN:   -fptrauth-calls -fptrauth-init-fini -fptrauth-init-fini-address-discrimination %s | \
 // RUN:   FileCheck %s --check-prefix=INITFINIADDR
 
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-indirect-gotos %s | FileCheck %s --check-prefix=GOTOS
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-calls -fptrauth-type-info-vtable-pointer-discrimination %s | \
+// RUN:   FileCheck %s --check-prefix=TYPEINFO
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-calls -fptrauth-function-pointer-type-discrimination %s | \
+// RUN:   FileCheck %s --check-prefix=FPTRTYPE
+
 // ALL: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
-// ALL: !{i32 1, !"aarch64-elf-pauthabi-version", i32 255}
+// ALL: !{i32 1, !"aarch64-elf-pauthabi-version", i32 3839}
 
 // INTRIN: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
 // INTRIN: !{i32 1, !"aarch64-elf-pauthabi-version", i32 1}
@@ -64,4 +80,13 @@
 // INITFINIADDR: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
 // INITFINIADDR: !{i32 1, !"aarch64-elf-pauthabi-version", i32 194}
 
+// GOTOS: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// GOTOS: !{i32 1, !"aarch64-elf-pauthabi-version", i32 512}
+
+// TYPEINFO: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// TYPEINFO: !{i32 1, !"aarch64-elf-pauthabi-version", i32 1026}
+
+// FPTRTYPE: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// FPTRTYPE: !{i32 1, !"aarch64-elf-pauthabi-version", i32 2050}
+
 void foo() {}
diff --git a/clang/test/CodeGen/aarch64-targetattr.c b/clang/test/CodeGen/aarch64-targetattr.c
index 4f891f938b61860..d6227be2ebef83d 100644
--- a/clang/test/CodeGen/aarch64-targetattr.c
+++ b/clang/test/CodeGen/aarch64-targetattr.c
@@ -195,19 +195,19 @@ void minusarch() {}
 // CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+fp-armv8,+lse,+neon,+ras,+rdm,+v8.1a,+v8.2a,+v8a" }
 // CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rdm,+sve,+v8.1a,+v8.2a,+v8a" }
 // CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rdm,+sve,+sve2,+v8.1a,+v8.2a,+v8a" }
-// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+ras,+rcpc,+rdm,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" }
-// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a710" "target-features"="+bf16,+complxnum,+crc,+dotprod,+ete,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+mte,+neon,+pauth,+perfmon,+ras,+rcpc,+rdm,+sb,+sve,+sve2,+sve2-bitperm,+trbe,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+v9a" }
+// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+predres,+ras,+rcpc,+rdm,+sb,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" }
+// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a710" "target-features"="+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+ete,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+mte,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+ssbs,+sve,+sve2,+sve2-bitperm,+trbe,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+v9a" }
 // CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "tune-cpu"="cortex-a710" }
 // CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+ete,+fp-armv8,+neon,+trbe,+v8a" }
 // CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "tune-cpu"="generic" }
 // CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fullfp16,+lse,+neon,+perfmon,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+v8.1a,+v8.2a,+v8a" "tune-cpu"="cortex-a710" }
 // CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+sve" "tune-cpu"="cortex-a710" }
-// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+ccdp,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
-// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+ccdp,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,-sve" }
+// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,-sve" }
 // CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+sve" }
 // CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16" }
-// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
-// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "branch-target-enforcement" "guarded-control-stack" "no-trapping-math"="true" "sign-return-address"="non-leaf" "sign-return-address-key"="a_key" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
+// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
+// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "branch-target-enforcement" "guarded-control-stack" "no-trapping-math"="true" "sign-return-address"="non-leaf" "sign-return-address-key"="a_key" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
 // CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 // CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-v9.3a" }
 //.
diff --git a/clang/test/CodeGen/bit-int-ubsan.c b/clang/test/CodeGen/bit-int-ubsan.c
new file mode 100644
index 000000000000000..40e4fde1d1f417f
--- /dev/null
+++ b/clang/test/CodeGen/bit-int-ubsan.c
@@ -0,0 +1,78 @@
+// REQUIRES: x86-registered-target
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -fsanitize=array-bounds,enum,float-cast-overflow,integer-divide-by-zero,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation,implicit-integer-sign-change,unsigned-integer-overflow,signed-integer-overflow,shift-base,shift-exponent -O0 -emit-llvm -o - %s | FileCheck %s
+
+// The runtime test checking the _BitInt ubsan feature is located in compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
+
+typedef unsigned int uint32_t;
+uint32_t float_divide_by_zero() {
+  float f = 1.0f / 0.0f;
+  // CHECK: constant { i16, i16, [8 x i8] } { i16 1, i16 32, [8 x i8] c"'float'\00" }
+  _BitInt(37) r = (_BitInt(37))f;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 13, [20 x i8] c"'_BitInt(37)'\00%\00\00\00\00\00" }
+  return r;
+}
+
+uint32_t integer_divide_by_zero() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = 1 / 0;
+  // CHECK: constant { i16, i16, [32 x i8] } { i16 0, i16 10, [32 x i8] c"'uint32_t' (aka 'unsigned int')\00" }
+  return x;
+}
+
+uint32_t implicit_unsigned_integer_truncation() {
+  unsigned _BitInt(37) x = 2U;
+  x += float_divide_by_zero();
+  x += integer_divide_by_zero();
+  x = x + 0xFFFFFFFFFFFFFFFFULL;
+  // CHECK: constant { i16, i16, [23 x i8] } { i16 0, i16 12, [23 x i8] c"'unsigned _BitInt(37)'\00" }
+  uint32_t r = x & 0xFFFFFFFF;
+  return r;
+}
+
+uint32_t array_bounds() {
+  _BitInt(37) x[4];
+  _BitInt(37) y = x[10];
+  // CHECK: constant { i16, i16, [17 x i8] } { i16 -1, i16 0, [17 x i8] c"'_BitInt(37)[4]'\00" }
+  return (uint32_t)y;
+}
+
+uint32_t float_cast_overflow() {
+  float a = 100000000.0f;
+  _BitInt(7) b = (_BitInt(7))a;
+  // CHECK: constant { i16, i16, [19 x i8] } { i16 2, i16 7, [19 x i8] c"'_BitInt(7)'\00\07\00\00\00\00\00" }
+  return b;
+}
+
+_BitInt(13) implicit_signed_integer_truncation() {
+  _BitInt(73) x = (_BitInt(73)) ~((~0UL) >> 1);
+  return x;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 {{([[:xdigit:]]{2})}}, [20 x i8] c"'_BitInt(73)'\00I\00\00\00\00\00" }
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 9, [20 x i8] c"'_BitInt(13)'\00\0D\00\00\00\00\00" }
+}
+
+uint32_t negative_shift1(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(9) c = -2;
+  return x >> c;
+  // CHECK: constant { i16, i16, [19 x i8] } { i16 2, i16 9, [19 x i8] c"'_BitInt(9)'\00\09\00\00\00\00\00" }
+}
+
+uint32_t negative_shift2(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(17) c = -2;
+  return x >> c;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 11, [20 x i8] c"'_BitInt(17)'\00\11\00\00\00\00\00" }
+}
+
+uint32_t negative_shift3(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(34) c = -2;
+  return x >> c;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 13, [20 x i8] c"'_BitInt(34)'\00\22\00\00\00\00\00" }
+}
+
+uint32_t negative_shift5(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(68) c = -2;
+  return x >> c;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 {{([[:xdigit:]]{2})}}, [20 x i8] c"'_BitInt(68)'\00D\00\00\00\00\00" }
+}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 75b9d6d1fe1902c..20399b73e637575 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -22,6 +22,9 @@
 // RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_86 -target-feature +ptx72 \
 // RUN:            -fcuda-is-device -emit-llvm -o - -x cuda %s \
 // RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX72_SM86 -check-prefix=LP64 %s
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_89 -target-feature +ptx81 \
+// RUN:            -fcuda-is-device -emit-llvm -o - -x cuda %s \
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX81_SM89 %s
 
 #define __device__ __attribute__((device))
 #define __global__ __attribute__((global))
@@ -968,6 +971,39 @@ __device__ void nvvm_cvt_sm80() {
   // CHECK: ret void
 }
 
+// CHECK-LABEL: nvvm_cvt_sm89
+__device__ void nvvm_cvt_sm89() {
+#if __CUDA_ARCH__ >= 890
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e4m3x2.rn(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e4m3x2_rn(1.0f, 1.0f);
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e4m3x2.rn.relu(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e4m3x2_rn_relu(1.0f, 1.0f);
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e5m2x2.rn(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e5m2x2_rn(1.0f, 1.0f);
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e5m2x2.rn.relu(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e5m2x2_rn_relu(1.0f, 1.0f);
+
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn(<2 x half> <half 0xH3C00, half 0xH3C00>)
+  __nvvm_f16x2_to_e4m3x2_rn({1.0f16, 1.0f16});
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn.relu(<2 x half> <half 0xH3C00, half 0xH3C00>)
+  __nvvm_f16x2_to_e4m3x2_rn_relu({1.0f16, 1.0f16});
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn(<2 x half> <half 0xH3C00, half 0xH3C00>)
+  __nvvm_f16x2_to_e5m2x2_rn({1.0f16, 1.0f16});
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn.relu(<2 x half> <half 0xH3C00, half 0xH3C00>)
+  __nvvm_f16x2_to_e5m2x2_rn_relu({1.0f16, 1.0f16});
+
+  // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e4m3x2.to.f16x2.rn(i16 18504)
+  __nvvm_e4m3x2_to_f16x2_rn(0x4848);
+  // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e4m3x2.to.f16x2.rn.relu(i16 18504)
+  __nvvm_e4m3x2_to_f16x2_rn_relu(0x4848);
+  // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn(i16 19532)
+  __nvvm_e5m2x2_to_f16x2_rn(0x4c4c);
+  // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn.relu(i16 19532)
+  __nvvm_e5m2x2_to_f16x2_rn_relu(0x4c4c);
+#endif
+  // CHECK: ret void
+}
+
 #define NAN32 0x7FBFFFFF
 #define NAN16 (__bf16)0x7FBF
 #define BF16 (__bf16)0.1f
diff --git a/clang/test/CodeGen/overflow-idiom-exclusion-fp.c b/clang/test/CodeGen/overflow-idiom-exclusion-fp.c
deleted file mode 100644
index 511a88cc7a28368..000000000000000
--- a/clang/test/CodeGen/overflow-idiom-exclusion-fp.c
+++ /dev/null
@@ -1,83 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-overflow-pattern-exclusion=all -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-overflow-pattern-exclusion=all -fwrapv -emit-llvm -o - %s | FileCheck %s
-
-// Check for potential false positives from patterns that _almost_ match classic overflow-dependent or overflow-prone code patterns
-extern unsigned a, b, c;
-extern int u, v, w;
-
-extern unsigned some(void);
-
-// Make sure all these still have handler paths, we shouldn't be excluding
-// instrumentation of any "near" patterns.
-// CHECK-LABEL: close_but_not_quite
-void close_but_not_quite(void) {
-  // CHECK: br i1{{.*}}handler.
-  if (a + b > a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a - b < a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a + b < a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a + b + 1 < a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  // CHECK: br i1{{.*}}handler.
-  if (a + b < a + 1)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (b >= a + b)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a + a < a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a + b == a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler
-  // Although this can never actually overflow we are still checking that the
-  // sanitizer instruments it.
-  while (--a)
-    some();
-}
-
-// cvise'd kernel code that caused problems during development
-typedef unsigned _size_t;
-typedef enum { FSE_repeat_none } FSE_repeat;
-typedef enum { ZSTD_defaultAllowed } ZSTD_defaultPolicy_e;
-FSE_repeat ZSTD_selectEncodingType_repeatMode;
-ZSTD_defaultPolicy_e ZSTD_selectEncodingType_isDefaultAllowed;
-_size_t ZSTD_NCountCost(void);
-
-// CHECK-LABEL: ZSTD_selectEncodingType
-// CHECK: br i1{{.*}}handler
-void ZSTD_selectEncodingType(void) {
-  _size_t basicCost =
-             ZSTD_selectEncodingType_isDefaultAllowed ? ZSTD_NCountCost() : 0,
-         compressedCost = 3 + ZSTD_NCountCost();
-  if (basicCost <= compressedCost)
-    ZSTD_selectEncodingType_repeatMode = FSE_repeat_none;
-}
-
-// CHECK-LABEL: function_calls
-void function_calls(void) {
-  // CHECK: br i1{{.*}}handler
-  if (some() + b < some())
-    c = 9;
-}
-
-// CHECK-LABEL: not_quite_a_negated_unsigned_const
-void not_quite_a_negated_unsigned_const(void) {
-  // CHECK: br i1{{.*}}handler
-  a = -b;
-}
diff --git a/clang/test/CodeGen/sanitize-ignorelist-mainfile.c b/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
index 84d58b4c018e3a2..419e0417c1d8296 100644
--- a/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
+++ b/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
@@ -1,8 +1,8 @@
 /// Test mainfile in a sanitizer special case list.
-// RUN: rm -rf %t && split-file %s %t && cd %t
-// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment a.c -o - | FileCheck %s --check-prefixes=CHECK,DEFAULT
-// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=a.list a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
-// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=b.list a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: rm -rf %t && split-file %s %t
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,DEFAULT
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=%t/a.list %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=%t/b.list %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
 
 //--- a.list
 mainfile:*a.c
diff --git a/clang/test/CodeGenCXX/amdgpu-kernel-arg-pointer-type.cpp b/clang/test/CodeGenCXX/amdgpu-kernel-arg-pointer-type.cpp
index c8fb12315d0eeca..8df1d75e9ec8fa4 100644
--- a/clang/test/CodeGenCXX/amdgpu-kernel-arg-pointer-type.cpp
+++ b/clang/test/CodeGenCXX/amdgpu-kernel-arg-pointer-type.cpp
@@ -6,6 +6,10 @@
 // The original test passes the result through opt O2, but that seems to introduce invalid
 // addrspace casts which are not being fixed as part of the present change.
 
+// COMMON: define{{.*}} amdgpu_kernel void @_Z6kernelv() #[[ATTR:[0-9]+]]
+__attribute__((amdgpu_kernel, amdgpu_flat_work_group_size(1, 256))) void
+    kernel() {}
+
 // COMMON-LABEL: define{{.*}} amdgpu_kernel void @_Z7kernel1Pi(ptr {{.*}} %x)
 // CHECK-NOT: ={{.*}} addrspacecast [[TYPE:.*]] addrspace(1)* %{{.*}} to ptr
 __attribute__((amdgpu_kernel)) void kernel1(int *x) {
@@ -81,3 +85,4 @@ __attribute__((amdgpu_kernel)) void kernel8(struct SS a) {
   *a.x += 3.f;
 }
 
+// COMMON: attributes #[[ATTR]] = { {{.*}}"amdgpu-flat-work-group-size"="1,256"{{.*}} }
diff --git a/clang/test/CodeGenCXX/mangle-ms-auto-return.cpp b/clang/test/CodeGenCXX/mangle-ms-auto-return.cpp
deleted file mode 100644
index 737c9c407f47037..000000000000000
--- a/clang/test/CodeGenCXX/mangle-ms-auto-return.cpp
+++ /dev/null
@@ -1,369 +0,0 @@
-// RUN: %clang_cc1 -std=c++17 -fms-compatibility-version=19.20 -emit-llvm %s -o - -fms-extensions -fdelayed-template-parsing -triple=x86_64-pc-windows-msvc | FileCheck %s
-
-struct StructA {};
-
-template<class T>
-auto AutoT() { return T(); }
-
-template<class T>
-const auto AutoConstT() { return T(); }
-
-template<class T>
-volatile auto AutoVolatileT() { return T(); }
-
-template<class T>
-const volatile auto AutoConstVolatileT() { return T(); }
-
-// The qualifiers of the return type should always be emitted even for void types.
-// Void types usually have their qualifers stripped in the mangled name for MSVC ABI.
-void test_template_auto_void() {
-  AutoT<void>();
-  // CHECK: call {{.*}} @"??$AutoT@X@@YA?A_PXZ"
-
-  AutoT<const void>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CBX@@YA?A_PXZ"
-
-  AutoT<volatile void>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CCX@@YA?A_PXZ"
-
-  AutoT<const volatile void>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CDX@@YA?A_PXZ"
-
-  AutoConstT<void>();
-  // CHECK: call {{.*}} @"??$AutoConstT@X@@YA?B_PXZ"
-
-  AutoVolatileT<void>();
-  // CHECK: call {{.*}} @"??$AutoVolatileT@X@@YA?C_PXZ"
-
-  AutoConstVolatileT<void>();
-  // CHECK: call {{.*}} @"??$AutoConstVolatileT@X@@YA?D_PXZ"
-}
-
-void test_template_auto_int() {
-  AutoT<int>();
-  // CHECK: call {{.*}} @"??$AutoT@H@@YA?A_PXZ"
-
-  AutoT<const int>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CBH@@YA?A_PXZ"
-
-  AutoT<volatile int>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CCH@@YA?A_PXZ"
-
-  AutoT<const volatile int>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CDH@@YA?A_PXZ"
-
-  AutoConstT<int>();
-  // CHECK: call {{.*}} @"??$AutoConstT@H@@YA?B_PXZ"
-
-  AutoVolatileT<int>();
-  // CHECK: call {{.*}} @"??$AutoVolatileT@H@@YA?C_PXZ"
-
-  AutoConstVolatileT<int>();
-  // CHECK: call {{.*}} @"??$AutoConstVolatileT@H@@YA?D_PXZ"
-}
-
-void test_template_auto_struct() {
-  AutoT<StructA>();
-  // CHECK: call {{.*}} @"??$AutoT@UStructA@@@@YA?A_PXZ"
-
-  AutoT<const StructA>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CBUStructA@@@@YA?A_PXZ"
-
-  AutoConstT<StructA>();
-  // CHECK: call {{.*}} @"??$AutoConstT@UStructA@@@@YA?B_PXZ"
-
-  AutoVolatileT<StructA>();
-  // CHECK: call {{.*}} @"??$AutoVolatileT@UStructA@@@@YA?C_PXZ"
-
-  AutoConstVolatileT<StructA>();
-  // CHECK: call {{.*}} @"??$AutoConstVolatileT@UStructA@@@@YA?D_PXZ"
-}
-
-void test_template_auto_ptr() {
-  AutoT<int*>();
-  // CHECK: call {{.*}} @"??$AutoT@PEAH@@YA?A_PXZ"
-
-  AutoT<const int*>();
-  // CHECK: call {{.*}} @"??$AutoT@PEBH@@YA?A_PXZ"
-
-  AutoT<const int* const>();
-  // CHECK: call {{.*}} @"??$AutoT@QEBH@@YA?A_PXZ"
-
-  AutoConstT<int*>();
-  // CHECK: call {{.*}} @"??$AutoConstT@PEAH@@YA?B_PXZ"
-
-  AutoVolatileT<int*>();
-  // CHECK: call {{.*}} @"??$AutoVolatileT@PEAH@@YA?C_PXZ"
-
-  AutoConstVolatileT<int*>();
-  // CHECK: call {{.*}} @"??$AutoConstVolatileT@PEAH@@YA?D_PXZ"
-}
-
-template<class T>
-auto* PtrAutoT() { return T(); }
-
-template<class T>
-const auto* PtrAutoConstT() { return T(); }
-
-template<class T>
-volatile auto* PtrAutoVolatileT() { return T(); }
-
-template<class T>
-const volatile auto* PtrAutoConstVolatileT() { return T(); }
-
-void test_template_ptr_auto() {
-  PtrAutoT<int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoT@PEAH@@YAPEA_PXZ"
-
-  PtrAutoT<const int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoT@PEBH@@YAPEA_PXZ"
-
-  PtrAutoT<const int* const>();
-  // CHECK: call {{.*}} @"??$PtrAutoT@QEBH@@YAPEA_PXZ"
-
-  PtrAutoConstT<int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoConstT@PEAH@@YAPEB_PXZ"
-
-  PtrAutoVolatileT<int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoVolatileT@PEAH@@YAPEC_PXZ"
-
-  PtrAutoConstVolatileT<int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoConstVolatileT@PEAH@@YAPED_PXZ"
-}
-
-int func_int();
-const int func_constint();
-void func_void();
-int* func_intptr();
-
-template<class T, T v>
-auto (*FuncPtrAutoT())() { return v; }
-
-void test_template_func_ptr_auto() {
-  FuncPtrAutoT<int (*)(), &func_int>();
-  // CHECK: call {{.*}} @"??$FuncPtrAutoT@P6AHXZ$1?func_int@@YAHXZ@@YAP6A?A_PXZXZ"
-
-  FuncPtrAutoT<const int (*)(), &func_constint>();
-  // CHECK: call {{.*}} @"??$FuncPtrAutoT@P6A?BHXZ$1?func_constint@@YA?BHXZ@@YAP6A?A_PXZXZ"
-
-  FuncPtrAutoT<void (*)(), &func_void>();
-  // CHECK: call {{.*}} @"??$FuncPtrAutoT@P6AXXZ$1?func_void@@YAXXZ@@YAP6A?A_PXZXZ"
-
-  FuncPtrAutoT<int * (*)(), &func_intptr>();
-  // CHECK: call {{.*}} @"??$FuncPtrAutoT@P6APEAHXZ$1?func_intptr@@YAPEAHXZ@@YAP6A?A_PXZXZ"
-}
-
-template<class T>
-auto& RefAutoT(T& x) { return x; }
-
-template<class T>
-const auto& ConstRefAutoT(T& x) { return x; }
-
-template<class T>
-auto&& RRefAutoT(T& x) { return static_cast<int&&>(x); }
-
-void test_template_ref_auto() {
-  int x;
-
-  RefAutoT(x);
-  // CHECK: call {{.*}} @"??$RefAutoT@H@@YAAEA_PAEAH@Z"
-
-  ConstRefAutoT(x);
-  // CHECK: call {{.*}} @"??$ConstRefAutoT@H@@YAAEB_PAEAH@Z"
-
-  RRefAutoT(x);
-  // CHECK: call {{.*}} @"??$RRefAutoT@H@@YA$$QEA_PAEAH@Z"
-}
-
-template<class T>
-decltype(auto) DecltypeAutoT() { return T(); }
-
-template<class T>
-decltype(auto) DecltypeAutoT2(T& x) { return static_cast<T&&>(x); }
-
-void test_template_decltypeauto() {
-  DecltypeAutoT<void>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@X@@YA?A_TXZ"
-
-  DecltypeAutoT<const void>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CBX@@YA?A_TXZ"
-
-  DecltypeAutoT<volatile void>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CCX@@YA?A_TXZ"
-
-  DecltypeAutoT<const volatile void>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CDX@@YA?A_TXZ"
-
-  DecltypeAutoT<int>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@H@@YA?A_TXZ"
-
-  DecltypeAutoT<const int>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CBH@@YA?A_TXZ"
-
-  DecltypeAutoT<volatile int>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CCH@@YA?A_TXZ"
-
-  DecltypeAutoT<const volatile int>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CDH@@YA?A_TXZ"
-
-  int x;
-
-  DecltypeAutoT2(x);
-  // CHECK: call {{.*}} @"??$DecltypeAutoT2@H@@YA?A_TAEAH@Z"
-}
-
-// Still want to use clang's custom mangling for lambdas to keep backwards compatibility until
-// MSVC lambda name mangling has been deciphered.
-void test_lambda() {
-  auto lambdaIntRetAuto = []() { return 0; };
-  lambdaIntRetAuto();
-  // CHECK: call {{.*}} @"??R<lambda_1>@?0??test_lambda@@YAXXZ@QEBA?A?<auto>@@XZ"
-
-  auto lambdaIntRet = []() -> int { return 0; };
-  lambdaIntRet();
-  // CHECK: call {{.*}} @"??R<lambda_2>@?0??test_lambda@@YAXXZ@QEBA@XZ"
-
-  auto lambdaGenericIntIntRetAuto = [](auto a) { return a; };
-  lambdaGenericIntIntRetAuto(0);
-  // CHECK: call {{.*}} @"??$?RH@<lambda_0>@?0??test_lambda@@YAXXZ@QEBA?A?<auto>@@H@Z"
-}
-
-auto TestTrailingInt() -> int {
-  return 0;
-}
-
-auto TestTrailingConstVolatileVoid() -> const volatile void {
-}
-
-auto TestTrailingStructA() -> StructA {
-  return StructA{};
-}
-
-void test_trailing_return() {
-  TestTrailingInt();
-  // CHECK: call {{.*}} @"?TestTrailingInt@@YAHXZ"
-
-  TestTrailingConstVolatileVoid();
-  // CHECK: call {{.*}} @"?TestTrailingConstVolatileVoid@@YAXXZ"
-
-  TestTrailingStructA();
-  // CHECK: call {{.*}} @"?TestTrailingStructA@@YA?AUStructA@@XZ"
-}
-
-auto TestNonTemplateAutoInt() {
-  return 0;
-}
-
-auto TestNonTemplateAutoVoid() {
-  return;
-}
-
-auto TestNonTemplateAutoStructA() {
-  return StructA{};
-}
-
-const auto TestNonTemplateConstAutoInt() {
-  return 0;
-}
-
-const auto TestNonTemplateConstAutoVoid() {
-  return;
-}
-
-const auto TestNonTemplateConstAutoStructA() {
-  return StructA{};
-}
-
-void test_nontemplate_auto() {
-  TestNonTemplateAutoInt();
-  // CHECK: call {{.*}} @"?TestNonTemplateAutoInt@@YA@XZ"
-
-  TestNonTemplateAutoVoid();
-  // CHECK: call {{.*}} @"?TestNonTemplateAutoVoid@@YA@XZ"
-
-  TestNonTemplateAutoStructA();
-  // CHECK: call {{.*}} @"?TestNonTemplateAutoStructA@@YA@XZ"
-
-  TestNonTemplateConstAutoInt();
-  // CHECK: call {{.*}} @"?TestNonTemplateConstAutoInt@@YA@XZ"
-
-  TestNonTemplateConstAutoVoid();
-  // CHECK: call {{.*}} @"?TestNonTemplateConstAutoVoid@@YA@XZ"
-
-  TestNonTemplateConstAutoStructA();
-  // CHECK: call {{.*}} @"?TestNonTemplateConstAutoStructA@@YA@XZ"
-}
-
-decltype(auto) TestNonTemplateDecltypeAutoInt() {
-    return 0;
-}
-
-decltype(auto) TestNonTemplateDecltypeAutoVoid() {
-    return;
-}
-
-decltype(auto) TestNonTemplateDecltypeAutoStructA() {
-    return StructA{};
-}
-
-void test_nontemplate_decltypeauto() {
-  TestNonTemplateDecltypeAutoInt();
-  // CHECK: call {{.*}} @"?TestNonTemplateDecltypeAutoInt@@YA@XZ"
-
-  TestNonTemplateDecltypeAutoVoid();
-  // CHECK: call {{.*}} @"?TestNonTemplateDecltypeAutoVoid@@YA@XZ"
-
-  TestNonTemplateDecltypeAutoStructA();
-  // CHECK: call {{.*}} @"?TestNonTemplateDecltypeAutoStructA@@YA@XZ"
-}
-
-struct StructB {
-  int x;
-};
-
-template<class T>
-auto StructB::* AutoMemberDataPtrT(T x) { return x; }
-
-template<class T>
-const auto StructB::* AutoConstMemberDataPtrT(T x) { return x; }
-
-void test_template_auto_member_data_ptr() {
-  AutoMemberDataPtrT(&StructB::x);
-  // CHECK: call {{.*}} @"??$AutoMemberDataPtrT@PEQStructB@@H@@YAPEQStructB@@_PPEQ0@H@Z"
-
-  AutoConstMemberDataPtrT(&StructB::x);
-  // CHECK: call {{.*}} @"??$AutoConstMemberDataPtrT@PEQStructB@@H@@YAPERStructB@@_PPEQ0@H@Z"
-}
-
-struct StructC {
-  void test() {}
-};
-
-struct StructD {
-  const int test() { return 0; }
-};
-
-template<class T>
-auto (StructC::*AutoMemberFuncPtrT(T x))() { return x; }
-
-template<class T>
-const auto (StructD::*AutoConstMemberFuncPtrT(T x))() { return x; }
-
-void test_template_auto_member_func_ptr() {
-  AutoMemberFuncPtrT(&StructC::test);
-  // CHECK: call {{.*}} @"??$AutoMemberFuncPtrT@P8StructC@@EAAXXZ@@YAP8StructC@@EAA?A_PXZP80@EAAXXZ@Z"
-
-  AutoConstMemberFuncPtrT(&StructD::test);
-  // CHECK: call {{.*}} @"??$AutoConstMemberFuncPtrT@P8StructD@@EAA?BHXZ@@YAP8StructD@@EAA?B_PXZP80@EAA?BHXZ@Z"
-}
-
-template<class T>
-auto * __attribute__((address_space(1))) * AutoPtrAddressSpaceT() {
-  T * __attribute__((address_space(1))) * p = nullptr;
-  return p;
-}
-
-void test_template_auto_address_space_ptr() {
-  AutoPtrAddressSpaceT<int>();
-  // CHECK: call {{.*}} @"??$AutoPtrAddressSpaceT@H@@YA?A?<auto>@@XZ"
-}
diff --git a/clang/test/CodeGenCXX/mangle-ms-auto-templates-memptrs.cpp b/clang/test/CodeGenCXX/mangle-ms-auto-templates-memptrs.cpp
index b7bc3953f0b4380..360ebdecc5562be 100644
--- a/clang/test/CodeGenCXX/mangle-ms-auto-templates-memptrs.cpp
+++ b/clang/test/CodeGenCXX/mangle-ms-auto-templates-memptrs.cpp
@@ -34,15 +34,15 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmTemplate@$I?f@V@@QEAAXXZA@A@@@QEAA@XZ"
 
   AutoFunc<&S::f>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MP8S@@EAAXXZ1?f@1@QEAAXXZ@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MP8S@@EAAXXZ1?f@1@QEAAXXZ@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$1?f@S@@QEAAXXZ@@YA?A?<auto>@@XZ"
 
   AutoFunc<&M::f>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MP8M@@EAAXXZH?f@1@QEAAXXZA@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MP8M@@EAAXXZH?f@1@QEAAXXZA@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$H?f@M@@QEAAXXZA@@@YA?A?<auto>@@XZ"
 
   AutoFunc<&V::f>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MP8V@@EAAXXZI?f@1@QEAAXXZA@A@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MP8V@@EAAXXZI?f@1@QEAAXXZA@A@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$I?f@V@@QEAAXXZA@A@@@YA?A?<auto>@@XZ"
 
   AutoParmTemplate<&S::a> auto_data_single_inheritance;
@@ -58,14 +58,14 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmTemplate@$FBA@A@@@QEAA@XZ"
 
   AutoFunc<&S::a>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQS@@H07@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQS@@H07@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$07@@YA?A?<auto>@@XZ"
 
   AutoFunc<&M::a>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQM@@H0M@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQM@@H0M@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$0M@@@YA?A?<auto>@@XZ"
 
   AutoFunc<&V::a>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQV@@HFBA@A@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQV@@HFBA@A@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$FBA@A@@@YA?A?<auto>@@XZ"
 }
diff --git a/clang/test/CodeGenCXX/mangle-ms-auto-templates-nullptr.cpp b/clang/test/CodeGenCXX/mangle-ms-auto-templates-nullptr.cpp
index 251d9219c01ce28..8f98c1e59f73d79 100644
--- a/clang/test/CodeGenCXX/mangle-ms-auto-templates-nullptr.cpp
+++ b/clang/test/CodeGenCXX/mangle-ms-auto-templates-nullptr.cpp
@@ -19,6 +19,6 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmTemplate@$0A@@@QEAA@XZ"
 
   AutoFunc<nullptr>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$M$$T0A@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$M$$T0A@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$0A@@@YA?A?<auto>@@XZ"
 }
diff --git a/clang/test/CodeGenCXX/mangle-ms-auto-templates.cpp b/clang/test/CodeGenCXX/mangle-ms-auto-templates.cpp
index effcc31ee311038..ff5395cea75eb77 100644
--- a/clang/test/CodeGenCXX/mangle-ms-auto-templates.cpp
+++ b/clang/test/CodeGenCXX/mangle-ms-auto-templates.cpp
@@ -26,7 +26,7 @@ int j;
 
 void template_mangling() {
   AutoFunc<1>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MH00@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MH00@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$00@@YA?A?<auto>@@XZ"
   AutoParmTemplate<0> auto_int;
   // AFTER: call {{.*}} @"??0?$AutoParmTemplate@$MH0A@@@QEAA@XZ"
@@ -52,7 +52,7 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmsTemplate@$00$0HPPPPPPPPPPPPPPP@@@QEAA@XZ"
 
   AutoFunc<&i>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MPEAH1?i@@3HA@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MPEAH1?i@@3HA@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$1?i@@3HA@@YA?A?<auto>@@XZ"
 
   AutoParmTemplate<&i> auto_int_ptr;
@@ -64,7 +64,7 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmsTemplate@$1?i@@3HA$1?j@@3HA@@QEAA@XZ"
 
   AutoFunc<&Func>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MP6AHXZ1?Func@@YAHXZ@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MP6AHXZ1?Func@@YAHXZ@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$1?Func@@YAHXZ@@YA?A?<auto>@@XZ"
 
   AutoParmTemplate<&Func> auto_func_ptr;
diff --git a/clang/test/CodeGenHLSL/default_validator_version.hlsl b/clang/test/CodeGenHLSL/default_validator_version.hlsl
new file mode 100644
index 000000000000000..907526dd685a966
--- /dev/null
+++ b/clang/test/CodeGenHLSL/default_validator_version.hlsl
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -xhlsl -o - %s | FileCheck %s
+
+// CHECK:!dx.valver = !{![[valver:[0-9]+]]}
+// CHECK:![[valver]] = !{i32 1, i32 8}
+
+float bar(float a, float b);
+
+float foo(float a, float b) {
+  return bar(a, b);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
index f44465746356878..138616ccca7182a 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
@@ -49,7 +49,7 @@ void test_s_wait_event_export_ready() {
 }
 
 // CHECK-LABEL: @test_global_add_f32
-// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}})
+// CHECK: = atomicrmw fadd ptr addrspace(1) %addr, float %x syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.ignore.denormal.mode !{{[0-9]+$}}
 void test_global_add_f32(float *rtn, global float *addr, float x) {
   *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x);
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx12.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx12.cl
index 63381942eaba574..6b8a6d14575db88 100644
--- a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx12.cl
+++ b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx12.cl
@@ -11,7 +11,7 @@ typedef short __attribute__((ext_vector_type(2))) short2;
 
 // CHECK-LABEL: test_local_add_2bf16
 // CHECK: [[BC0:%.+]] = bitcast <2 x i16> {{.+}} to <2 x bfloat>
-// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] seq_cst, align 4
+// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] syncscope("agent") monotonic, align 4
 // CHECK-NEXT: bitcast <2 x bfloat> [[RMW]] to <2 x i16>
 
 // GFX12-LABEL:  test_local_add_2bf16
@@ -22,7 +22,7 @@ short2 test_local_add_2bf16(__local short2 *addr, short2 x) {
 
 // CHECK-LABEL: test_local_add_2bf16_noret
 // CHECK: [[BC0:%.+]] = bitcast <2 x i16> {{.+}} to <2 x bfloat>
-// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] seq_cst, align 4
+// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] syncscope("agent") monotonic, align 4
 // CHECK-NEXT: bitcast <2 x bfloat> [[RMW]] to <2 x i16>
 
 // GFX12-LABEL:  test_local_add_2bf16_noret
@@ -32,7 +32,7 @@ void test_local_add_2bf16_noret(__local short2 *addr, short2 x) {
 }
 
 // CHECK-LABEL: test_local_add_2f16
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} monotonic, align 4
 // GFX12-LABEL:  test_local_add_2f16
 // GFX12: ds_pk_add_rtn_f16
 half2 test_local_add_2f16(__local half2 *addr, half2 x) {
@@ -40,7 +40,7 @@ half2 test_local_add_2f16(__local half2 *addr, half2 x) {
 }
 
 // CHECK-LABEL: test_local_add_2f16_noret
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} monotonic, align 4
 // GFX12-LABEL:  test_local_add_2f16_noret
 // GFX12: ds_pk_add_f16
 void test_local_add_2f16_noret(__local half2 *addr, half2 x) {
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx8.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx8.cl
index ad4d0b7af3d4be7..2f00977ec6014e1 100644
--- a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx8.cl
+++ b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx8.cl
@@ -6,7 +6,7 @@
 // REQUIRES: amdgpu-registered-target
 
 // CHECK-LABEL: test_fadd_local
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, float %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, float %{{.+}} monotonic, align 4
 // GFX8-LABEL: test_fadd_local$local:
 // GFX8: ds_add_rtn_f32 v2, v0, v1
 // GFX8: s_endpgm
@@ -16,7 +16,7 @@ kernel void test_fadd_local(__local float *ptr, float val){
 }
 
 // CHECK-LABEL: test_fadd_local_volatile
-// CHECK: = atomicrmw volatile fadd ptr addrspace(3) %{{.+}}, float %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw volatile fadd ptr addrspace(3) %{{.+}}, float %{{.+}} monotonic, align 4
 kernel void test_fadd_local_volatile(volatile __local float *ptr, float val){
     volatile float *res;
     *res = __builtin_amdgcn_ds_atomic_fadd_f32(ptr, val);
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl
index e2117f11858f7f8..c525c250c937cad 100644
--- a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl
+++ b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl
@@ -9,7 +9,7 @@
 typedef half __attribute__((ext_vector_type(2))) half2;
 
 // CHECK-LABEL: test_global_add_f64
-// CHECK: call double @llvm.amdgcn.global.atomic.fadd.f64.p1.f64(ptr addrspace(1) %{{.*}}, double %{{.*}})
+// CHECK: = atomicrmw fadd ptr addrspace(1) %{{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}
 // GFX90A-LABEL:  test_global_add_f64$local:
 // GFX90A:  global_atomic_add_f64
 void test_global_add_f64(__global double *addr, double x) {
@@ -99,7 +99,7 @@ void test_flat_global_max_f64(__global double *addr, double x){
 }
 
 // CHECK-LABEL: test_ds_add_local_f64
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, double %{{.+}} seq_cst, align 8
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, double %{{.+}} monotonic, align 8
 // GFX90A:  test_ds_add_local_f64$local
 // GFX90A:  ds_add_rtn_f64
 void test_ds_add_local_f64(__local double *addr, double x){
@@ -108,7 +108,7 @@ void test_ds_add_local_f64(__local double *addr, double x){
 }
 
 // CHECK-LABEL: test_ds_addf_local_f32
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, float %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, float %{{.+}} monotonic, align 4
 // GFX90A-LABEL:  test_ds_addf_local_f32$local
 // GFX90A:  ds_add_rtn_f32
 void test_ds_addf_local_f32(__local float *addr, float x){
@@ -117,7 +117,7 @@ void test_ds_addf_local_f32(__local float *addr, float x){
 }
 
 // CHECK-LABEL: @test_global_add_f32
-// CHECK: call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}})
+// CHECK: = atomicrmw fadd ptr addrspace(1) %{{.+}}, float %{{.+}} syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.ignore.denormal.mode !{{[0-9]+$}}
 void test_global_add_f32(float *rtn, global float *addr, float x) {
   *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x);
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx940.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx940.cl
index 92a33ceac2290f5..5481138b9fee43a 100644
--- a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx940.cl
+++ b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx940.cl
@@ -44,7 +44,7 @@ short2 test_global_add_2bf16(__global short2 *addr, short2 x) {
 // CHECK-LABEL: test_local_add_2bf16
 
 // CHECK: [[BC0:%.+]] = bitcast <2 x i16> {{.+}} to <2 x bfloat>
-// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] seq_cst, align 4
+// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] syncscope("agent") monotonic, align 4{{$}}
 // CHECK-NEXT: bitcast <2 x bfloat> [[RMW]] to <2 x i16>
 
 // GFX940-LABEL:  test_local_add_2bf16
@@ -54,7 +54,7 @@ short2 test_local_add_2bf16(__local short2 *addr, short2 x) {
 }
 
 // CHECK-LABEL: test_local_add_2f16
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} monotonic, align 4
 // GFX940-LABEL:  test_local_add_2f16
 // GFX940: ds_pk_add_rtn_f16
 half2 test_local_add_2f16(__local half2 *addr, half2 x) {
@@ -62,7 +62,7 @@ half2 test_local_add_2f16(__local half2 *addr, half2 x) {
 }
 
 // CHECK-LABEL: test_local_add_2f16_noret
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} monotonic, align 4
 // GFX940-LABEL:  test_local_add_2f16_noret
 // GFX940: ds_pk_add_f16
 void test_local_add_2f16_noret(__local half2 *addr, half2 x) {
@@ -70,7 +70,7 @@ void test_local_add_2f16_noret(__local half2 *addr, half2 x) {
 }
 
 // CHECK-LABEL: @test_global_add_f32
-// CHECK: call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}})
+// CHECK: = atomicrmw fadd ptr addrspace(1) %{{.+}}, float %{{.+}} syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.ignore.denormal.mode !{{[0-9]+$}}
 void test_global_add_f32(float *rtn, global float *addr, float x) {
   *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x);
 }
diff --git a/clang/test/Driver/aarch64-negative-modifiers-for-default-features.c b/clang/test/Driver/aarch64-negative-modifiers-for-default-features.c
new file mode 100644
index 000000000000000..03dd8af7588cae1
--- /dev/null
+++ b/clang/test/Driver/aarch64-negative-modifiers-for-default-features.c
@@ -0,0 +1,12 @@
+// Test that default features (e.g. flagm/sb/ssbs for 8.5) can be disabled via -march.
+
+// RUN: %clang --target=aarch64 -march=armv8.5-a+noflagm+nosb+nossbs -c %s -### 2>&1 | FileCheck %s
+// CHECK: "-triple" "aarch64"
+// CHECK-SAME: "-target-feature" "+v8.5a"
+// CHECK-SAME: "-target-feature" "-flagm"
+// CHECK-SAME: "-target-feature" "-sb"
+// CHECK-SAME: "-target-feature" "-ssbs"
+
+// CHECK-NOT: "-target-feature" "+flagm"
+// CHECK-NOT: "-target-feature" "+sb"
+// CHECK-NOT: "-target-feature" "+ssbs"
diff --git a/clang/test/Driver/arm-sb.c b/clang/test/Driver/arm-sb.c
index f2704f33c271113..9c0f381171cb6df 100644
--- a/clang/test/Driver/arm-sb.c
+++ b/clang/test/Driver/arm-sb.c
@@ -11,6 +11,6 @@
 
 // RUN: %clang -### -target arm-none-none-eabi %s 2>&1 | FileCheck %s --check-prefix=ABSENT
 // RUN: %clang -### -target aarch64-none-elf %s 2>&1 | FileCheck %s --check-prefix=ABSENT
-// RUN: %clang -### -target aarch64-none-elf -march=armv8.5a+nosb %s 2>&1 | FileCheck %s --check-prefix=ABSENT
+// RUN: %clang -### -target aarch64-none-elf -march=armv8.5a+nosb %s 2>&1 | FileCheck %s --check-prefix=NOSB
 // ABSENT-NOT: "-target-feature" "+sb"
 // ABSENT-NOT: "-target-feature" "-sb"
diff --git a/clang/test/Driver/arm-target-as-mimplicit-it.s b/clang/test/Driver/arm-target-as-mimplicit-it.s
index d30433ab374d598..e40206dfc3a2938 100644
--- a/clang/test/Driver/arm-target-as-mimplicit-it.s
+++ b/clang/test/Driver/arm-target-as-mimplicit-it.s
@@ -30,7 +30,7 @@
 
 /// Test invalid input.
 // RUN: not %clang -target arm-linux-gnueabi -### -Wa,-mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=INVALID
-// RUN: not %clang -target arm-linux-gnueabi -### -Xassembler -mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=XINVALID
+// RUN: not %clang -target arm-linux-gnueabi -### -Xassembler -mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=INVALID
 // RUN: not %clang -target arm-linux-gnueabi -### -Wa,-mimplicit-it=always -Wa,-mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=INVALID
 // RUN: not %clang -target arm-linux-gnueabi -### -Wa,-mimplicit-it=always,-mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=INVALID
 
@@ -47,5 +47,4 @@
 // NEVER-NOT: "-arm-implicit-it={{.*}}"
 // ARM: "-mllvm" "-arm-implicit-it=arm"
 // THUMB: "-mllvm" "-arm-implicit-it=thumb"
-// INVALID: error: unsupported argument '-mimplicit-it=foo' to option '-Wa,'
-// XINVALID: error: unsupported argument '-mimplicit-it=foo' to option '-Xassembler'
+// INVALID: error: unsupported argument 'foo' to option '-Wa,-mimplicit-it='
diff --git a/clang/test/Driver/crel.c b/clang/test/Driver/crel.c
index a47e7ebfeedd88f..e1b3f58c6ea5b73 100644
--- a/clang/test/Driver/crel.c
+++ b/clang/test/Driver/crel.c
@@ -24,8 +24,10 @@
 
 /// The --allow-experimental-crel error check doesn't apply to LTO.
 // RUN: %clang -### --target=x86_64-linux -Werror -flto -Wa,--crel %s 2>&1 | FileCheck %s --check-prefix=LTO
+// RUN: %clang -### --target=x86_64-linux -Werror -flto -Wa,--crel -Wa,--no-crel %s 2>&1 | FileCheck %s --check-prefix=LTO-NO
 
 // LTO: "-plugin-opt=-crel"
+// LTO-NO-NOT: "-plugin-opt=-crel"
 
 // RUN: touch %t.o
 // RUN: not %clang -### --target=mips64-linux-gnu -flto -Wa,--crel %t.o 2>&1 | FileCheck %s --check-prefix=ERR
diff --git a/clang/test/Driver/darwin-print-target-triple.c b/clang/test/Driver/darwin-print-target-triple.c
deleted file mode 100644
index 4f5fdfe9d0db340..000000000000000
--- a/clang/test/Driver/darwin-print-target-triple.c
+++ /dev/null
@@ -1,42 +0,0 @@
-// Test the output of -print-target-triple on Darwin.
-// See https://github.com/llvm/llvm-project/issues/61762
-
-//
-// All platforms
-//
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=x86_64-apple-macos -mmacos-version-min=15 \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-MACOS %s
-// CHECK-CLANGRT-MACOS: x86_64-apple-macosx15.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=arm64-apple-ios -mios-version-min=9 \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-IOS %s
-// CHECK-CLANGRT-IOS: arm64-apple-ios9.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=arm64-apple-watchos -mwatchos-version-min=3 \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-WATCHOS %s
-// CHECK-CLANGRT-WATCHOS: arm64-apple-watchos3.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=armv7k-apple-watchos -mwatchos-version-min=3 \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-WATCHOS-ARMV7K %s
-// CHECK-CLANGRT-WATCHOS-ARMV7K: thumbv7-apple-watchos3.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=arm64-apple-tvos -mtvos-version-min=1\
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-TVOS %s
-// CHECK-CLANGRT-TVOS: arm64-apple-tvos1.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=arm64-apple-driverkit \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-DRIVERKIT %s
-// CHECK-CLANGRT-DRIVERKIT: arm64-apple-driverkit19.0.0
diff --git a/clang/test/Driver/flags.c b/clang/test/Driver/flags.c
index 16b760609c36dd3..7d2bef63fca3e17 100644
--- a/clang/test/Driver/flags.c
+++ b/clang/test/Driver/flags.c
@@ -33,3 +33,6 @@
 //
 // RUN: %clang --target=riscv64 -### -S -mno-implicit-float %s 2>&1 | FileCheck -check-prefix=TEST11 %s
 // TEST11: "-no-implicit-float"
+//
+// RUN: %clang --target=ppc64le -### -S -mno-red-zone %s 2>&1 | FileCheck -check-prefix=TEST12 %s
+// TEST12: "-disable-red-zone"
diff --git a/clang/test/Driver/hip-toolchain-rdc.hip b/clang/test/Driver/hip-toolchain-rdc.hip
index 7e6697a0e254f6c..ec79bf06afb92c2 100644
--- a/clang/test/Driver/hip-toolchain-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-rdc.hip
@@ -20,6 +20,29 @@
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,MSVC %s
 
+// Test fatbin symbol search for -l libraries.
+
+// RUN: touch librdctest.a rdctest2.bin
+// RUN: %clang -### --target=x86_64-linux-gnu -v \
+// RUN:   -fgpu-rdc -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -L. -lrdctest -l:rdctest2.bin \
+// RUN:   %s \
+// RUN: 2>&1 | FileCheck -check-prefixes=LIB,LNX-LIB %s
+// RUN: rm librdctest.a rdctest2.bin
+
+// RUN: touch rdctest.lib rdctest2.bin
+// RUN: %clang -### --target=x86_64-pc-windows-msvc -v \
+// RUN:   -fgpu-rdc -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -L. -lrdctest -l:rdctest2.bin \
+// RUN:   %s \
+// RUN: 2>&1 | FileCheck -check-prefixes=LIB,MSVC-LIB %s
+// RUN: rm rdctest.lib rdctest2.bin
+
+// LIB: HIP fatbin symbol search uses library path: .
+// LIB: HIP fatbin symbol search found library: .{{/|\\}}rdctest2.bin
+// LNX-LIB: HIP fatbin symbol search found library: .{{/|\\}}librdctest.a
+// MSVC-LIB: HIP fatbin symbol search found library: .{{/|\\}}rdctest.lib
+
 // check HIP fatbin and gpubin handle symbols and code object alignment in dumped llvm-mc input
 // CHECK: Found undefined HIP fatbin symbol: __hip_fatbin_[[ID1:[0-9a-f]+]]
 // CHECK: Found undefined HIP fatbin symbol: __hip_fatbin_[[ID2:[0-9a-f]+]]
diff --git a/clang/test/Driver/msse2avx.c b/clang/test/Driver/msse2avx.c
index a63ac9a6c866815..829b1b2b79360d4 100644
--- a/clang/test/Driver/msse2avx.c
+++ b/clang/test/Driver/msse2avx.c
@@ -1,7 +1,7 @@
-// RUN: %clang -### -c -target x86_64 -march=x86-64 -Xassembler -msse2avx %s 2>&1 | FileCheck  %s
-// RUN: %clang -### -c -target x86_64 -march=x86-64 -x assembler -Xassembler -msse2avx %s 2>&1 | FileCheck %s
+// RUN: %clang -### -c --target=x86_64 -march=x86-64 -Wa,-msse2avx %s 2>&1 | FileCheck  %s
+// RUN: %clang -### -c --target=x86_64 -march=x86-64 -x assembler -Xassembler -msse2avx %s 2>&1 | FileCheck %s
 
 // CHECK: "-msse2avx"
 
-// RUN: not %clang -### -c -target aarch64 -march=armv8a -msse2avx %s 2>&1 | FileCheck --check-prefix=ERR %s
-// ERR:   error: unsupported option '-msse2avx' for target 'aarch64'
+// RUN: not %clang -### -c --target=aarch64 -march=armv8a -Wa,-msse2avx %s 2>&1 | FileCheck --check-prefix=ERR %s
+// ERR:   error: unsupported argument '-msse2avx' to option '-Wa,'
diff --git a/clang/test/Driver/ohos.c b/clang/test/Driver/ohos.c
index 8de4e6de57f7ff8..af8218fb232e601 100644
--- a/clang/test/Driver/ohos.c
+++ b/clang/test/Driver/ohos.c
@@ -7,7 +7,6 @@
 // RUN:     --sysroot=%S/Inputs/ohos_native_tree/sysroot -fuse-ld=ld -march=armv7-a -mcpu=cortex-a7 -mfloat-abi=soft 2>&1 \
 // RUN:     | FileCheck -check-prefixes=CHECK,CHECK-ARM-A7-SOFT %s
 // CHECK: {{.*}}clang{{.*}}" "-cc1"
-// CHECK-NOT: "--mrelax-relocations"
 // CHECK-NOT: "-munwind-tables"
 // CHECK: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK: "-isysroot" "[[SYSROOT:[^"]+]]"
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a12.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a12.c
index 27f066a3107086a..fcc8b674df33f44 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a12.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a12.c
@@ -6,7 +6,6 @@
 // CHECK-NEXT:     Architecture Feature(s)                                Description
 // CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
 // CHECK-NEXT:     FEAT_FCMA                                              Enable Armv8.3-A Floating-point complex number support
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a13.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a13.c
index 197b21025995106..dae95b1297e1451 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a13.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a13.c
@@ -7,7 +7,6 @@
 // CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
 // CHECK-NEXT:     FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
 // CHECK-NEXT:     FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a14.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a14.c
index f1731ef034a0c1f..8ddcddede4110d9 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a14.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a14.c
@@ -7,7 +7,6 @@
 // CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
 // CHECK-NEXT:     FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a15.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a15.c
index dec48bb70331141..b3f0acefd1e2d48 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a15.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a15.c
@@ -10,7 +10,6 @@
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
 // CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
 // CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a16.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a16.c
index 477652d83d82cbb..6f417c1592b06cd 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a16.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a16.c
@@ -10,7 +10,6 @@
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
 // CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
 // CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a17.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a17.c
index 311cc94acddc96e..39e5b70c25d8860 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a17.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a17.c
@@ -10,7 +10,6 @@
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
 // CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
 // CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-m4.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-m4.c
index 44d618afef40689..29d66dc8826a77f 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-m4.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-m4.c
@@ -10,7 +10,6 @@
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
 // CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
 // CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
@@ -52,7 +51,6 @@
 // CHECK-NEXT:     FEAT_SME_F64F64                                        Enable Scalable Matrix Extension (SME) F64F64 instructions
 // CHECK-NEXT:     FEAT_SME_I16I64                                        Enable Scalable Matrix Extension (SME) I16I64 instructions
 // CHECK-NEXT:     FEAT_SPECRES                                           Enable Armv8.5-A execution and data prediction invalidation instructions
-// CHECK-NEXT:     FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
 // CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
 // CHECK-NEXT:     FEAT_TRF                                               Enable Armv8.4-A Trace extension
 // CHECK-NEXT:     FEAT_UAO                                               Enable Armv8.2-A UAO PState
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82.c b/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82.c
index 2b85201c2c6fe61..9875c6922d379a9 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82.c
@@ -5,7 +5,6 @@
 // CHECK-EMPTY:
 // CHECK-NEXT:     Architecture Feature(s)                                Description
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82ae.c b/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82ae.c
index 417687b4af287d4..2db44d7827aadb7 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82ae.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82ae.c
@@ -5,7 +5,6 @@
 // CHECK-EMPTY:
 // CHECK-NEXT:     Architecture Feature(s)                                Description
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-multi-selection-flags.c b/clang/test/Driver/print-multi-selection-flags.c
index 9f58d1b557fd74d..2770a3ad5eaa1dd 100644
--- a/clang/test/Driver/print-multi-selection-flags.c
+++ b/clang/test/Driver/print-multi-selection-flags.c
@@ -58,3 +58,18 @@
 // RUN: %clang -print-multi-flags-experimental --target=aarch64-none-elf -march=armv9-a            | FileCheck --check-prefix=CHECK-SVE2 %s
 // CHECK-SVE2: --target=aarch64-unknown-none-elf
 // CHECK-SVE2: -march=armv{{.*}}-a{{.*}}+simd{{.*}}+sve{{.*}}+sve2{{.*}}
+
+// RUN: %clang -print-multi-flags-experimental --target=riscv32-none-elf -march=rv32g | FileCheck --check-prefix=CHECK-RV32 %s
+// CHECK-RV32: --target=riscv32-unknown-none-elf
+// CHECK-RV32: -mabi=ilp32d
+// CHECK-RV32: -march=rv32i{{[0-9]+p[0-9]+}}_m{{[0-9]+p[0-9]+}}_a{{[0-9]+p[0-9]+}}_f{{[0-9]+p[0-9]+}}_d{{[0-9]+p[0-9]+}}_zicsr{{[0-9]+p[0-9]+}}_zifencei{{[0-9]+p[0-9]+}}_zmmul{{[0-9]+p[0-9]+}}
+
+// RUN: %clang -print-multi-flags-experimental --target=riscv64-none-elf -march=rv64g | FileCheck --check-prefix=CHECK-RV64 %s
+// CHECK-RV64: --target=riscv64-unknown-none-elf
+// CHECK-RV64: -mabi=lp64d
+// CHECK-RV64: -march=rv64i{{[0-9]+p[0-9]+}}_m{{[0-9]+p[0-9]+}}_a{{[0-9]+p[0-9]+}}_f{{[0-9]+p[0-9]+}}_d{{[0-9]+p[0-9]+}}_zicsr{{[0-9]+p[0-9]+}}_zifencei{{[0-9]+p[0-9]+}}_zmmul{{[0-9]+p[0-9]+}}
+
+// RUN: %clang -print-multi-flags-experimental --target=riscv32-none-elf -march=rv32e_zicsr_c | FileCheck --check-prefix=CHECK-RV32E-ORDER %s
+// CHECK-RV32E-ORDER: --target=riscv32-unknown-none-elf
+// CHECK-RV32E-ORDER: -mabi=ilp32e
+// CHECK-RV32E-ORDER: -march=rv32e{{[0-9]+p[0-9]+}}_c{{[0-9]+p[0-9]+}}_zicsr{{[0-9]+p[0-9]+}}
diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 91f12b8416b2a42..132422393170a90 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -172,6 +172,8 @@
 // CHECK-NEXT:     zicfiss              1.0       'Zicfiss' (Shadow stack)
 // CHECK-NEXT:     zacas                1.0       'Zacas' (Atomic Compare-And-Swap Instructions)
 // CHECK-NEXT:     zalasr               0.1       'Zalasr' (Load-Acquire and Store-Release Instructions)
+// CHECK-NEXT:     zvbc32e              0.7       'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements)
+// CHECK-NEXT:     zvkgs                0.7       'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)
 // CHECK-NEXT:     smmpm                1.0       'Smmpm' (Machine-level Pointer Masking for M-mode)
 // CHECK-NEXT:     smnpm                1.0       'Smnpm' (Machine-level Pointer Masking for next lower privilege mode)
 // CHECK-NEXT:     ssnpm                1.0       'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode)
diff --git a/clang/test/Driver/relax.c b/clang/test/Driver/relax.c
index 48d22fede12d761..93249cac7f40f88 100644
--- a/clang/test/Driver/relax.c
+++ b/clang/test/Driver/relax.c
@@ -1,4 +1,13 @@
-// RUN: %clang -### -c -integrated-as -Wa,--mrelax-relocations=no %s 2>&1 | FileCheck  %s
+// RUN: %clang -### --target=x86_64 -c -Wa,--mrelax-relocations=no %s 2>&1 | FileCheck  %s
 
 // CHECK: "-cc1"
 // CHECK: "-mrelax-relocations=no"
+
+// RUN: not %clang -### --target=x86_64 -c -Wa,-mrelax-relocations=x %s 2>&1 | FileCheck %s --check-prefix=ERR
+// ERR: error: unsupported argument 'x' to option '-Wa,-mrelax-relocations='
+
+// RUN: not %clang -### --target=aarch64 -c -Wa,-mrelax-relocations=no %s 2>&1 | FileCheck %s --check-prefix=ERR2
+// ERR2: error: unsupported argument '-mrelax-relocations=no' to option '-Wa,'
+
+// RUN: not %clang -### --target=x86_64-apple-darwin -c -Wa,-mrelax-relocations=no %s 2>&1 | FileCheck %s --check-prefix=ERR3
+// ERR3: error: unsupported option '-Wa,-mrelax-relocations=' for target 'x86_64-apple-darwin'
diff --git a/clang/test/Driver/riscv-sdata-warning.c b/clang/test/Driver/riscv-sdata-warning.c
deleted file mode 100644
index ace9a32ee116fd8..000000000000000
--- a/clang/test/Driver/riscv-sdata-warning.c
+++ /dev/null
@@ -1,4 +0,0 @@
-// REQUIRES: riscv-registered-target
-// RUN: %clang -S --target=riscv32-unknown-elf -fpic -msmall-data-limit=8 %s 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHECK-PIC-SDATA %s
-// CHECK-PIC-SDATA: warning: ignoring '-msmall-data-limit=' with -mcmodel=large for -fpic or RV64
diff --git a/clang/test/Driver/riscv-sdata.c b/clang/test/Driver/riscv-sdata.c
new file mode 100644
index 000000000000000..84cd109813db4ca
--- /dev/null
+++ b/clang/test/Driver/riscv-sdata.c
@@ -0,0 +1,5 @@
+// RUN: %clang -### -S --target=riscv64 %s 2>&1 | FileCheck %s
+// RUN: %clang -### -S --target=riscv64 -msmall-data-limit=8 %s 2>&1 | FileCheck %s --check-prefix=EIGHT
+
+// CHECK-NOT: "-msmall-data-limit"
+// EIGHT: "-msmall-data-limit" "8"
diff --git a/clang/test/Driver/sparc-target-features.c b/clang/test/Driver/sparc-target-features.c
index b36f63e7660e830..a839604ff1bc022 100644
--- a/clang/test/Driver/sparc-target-features.c
+++ b/clang/test/Driver/sparc-target-features.c
@@ -32,3 +32,6 @@
 // RUN: %clang --target=sparc -msoft-quad-float %s -### 2>&1 | FileCheck -check-prefix=SOFT-QUAD-FLOAT %s
 // HARD-QUAD-FLOAT: "-target-feature" "+hard-quad-float"
 // SOFT-QUAD-FLOAT: "-target-feature" "-hard-quad-float"
+
+// RUN: %clang --target=sparc -mv8plus %s -### 2>&1 | FileCheck -check-prefix=V8PLUS %s
+// V8PLUS: "-target-feature" "+v8plus"
diff --git a/clang/test/ExtractAPI/inherited_availability.m b/clang/test/ExtractAPI/inherited_availability.m
new file mode 100644
index 000000000000000..c24e7fa8e208f03
--- /dev/null
+++ b/clang/test/ExtractAPI/inherited_availability.m
@@ -0,0 +1,175 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -extract-api --pretty-sgf --emit-sgf-symbol-labels-for-testing -triple arm64-apple-macosx \
+// RUN:   -x objective-c-header %s -o %t/output.symbols.json -verify
+
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix A
+__attribute__((availability(macos, introduced=9.0, deprecated=12.0, obsoleted=20.0)))
+@interface A
+// A-LABEL: "!testLabel": "c:objc(cs)A"
+// A:      "availability": [
+// A-NEXT:   {
+// A-NEXT:     "deprecated": {
+// A-NEXT:       "major": 12,
+// A-NEXT:       "minor": 0,
+// A-NEXT:       "patch": 0
+// A-NEXT:     }
+// A-NEXT:     "domain": "macos"
+// A-NEXT:     "introduced": {
+// A-NEXT:       "major": 9,
+// A-NEXT:       "minor": 0,
+// A-NEXT:       "patch": 0
+// A-NEXT:     }
+// A-NEXT:     "obsoleted": {
+// A-NEXT:       "major": 20,
+// A-NEXT:       "minor": 0,
+// A-NEXT:       "patch": 0
+// A-NEXT:     }
+// A-NEXT:   }
+// A-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix CP
+@property(class) int CP;
+// CP-LABEL: "!testLabel": "c:objc(cs)A(cpy)CP"
+// CP:      "availability": [
+// CP-NEXT:   {
+// CP-NEXT:     "deprecated": {
+// CP-NEXT:       "major": 12,
+// CP-NEXT:       "minor": 0,
+// CP-NEXT:       "patch": 0
+// CP-NEXT:     }
+// CP-NEXT:     "domain": "macos"
+// CP-NEXT:     "introduced": {
+// CP-NEXT:       "major": 9,
+// CP-NEXT:       "minor": 0,
+// CP-NEXT:       "patch": 0
+// CP-NEXT:     }
+// CP-NEXT:     "obsoleted": {
+// CP-NEXT:       "major": 20,
+// CP-NEXT:       "minor": 0,
+// CP-NEXT:       "patch": 0
+// CP-NEXT:     }
+// CP-NEXT:   }
+// CP-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix IP
+@property int IP;
+// IP-LABEL: "!testLabel": "c:objc(cs)A(py)IP"
+// IP:      "availability": [
+// IP-NEXT:   {
+// IP-NEXT:     "deprecated": {
+// IP-NEXT:       "major": 12,
+// IP-NEXT:       "minor": 0,
+// IP-NEXT:       "patch": 0
+// IP-NEXT:     }
+// IP-NEXT:     "domain": "macos"
+// IP-NEXT:     "introduced": {
+// IP-NEXT:       "major": 9,
+// IP-NEXT:       "minor": 0,
+// IP-NEXT:       "patch": 0
+// IP-NEXT:     }
+// IP-NEXT:     "obsoleted": {
+// IP-NEXT:       "major": 20,
+// IP-NEXT:       "minor": 0,
+// IP-NEXT:       "patch": 0
+// IP-NEXT:     }
+// IP-NEXT:   }
+// IP-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix MR
+@property int moreRestrictive __attribute__((availability(macos, introduced=10.0, deprecated=11.0, obsoleted=19.0)));
+// MR-LABEL: "!testLabel": "c:objc(cs)A(py)moreRestrictive"
+// MR:      "availability": [
+// MR-NEXT:   {
+// MR-NEXT:     "deprecated": {
+// MR-NEXT:       "major": 11,
+// MR-NEXT:       "minor": 0,
+// MR-NEXT:       "patch": 0
+// MR-NEXT:     }
+// MR-NEXT:     "domain": "macos"
+// MR-NEXT:     "introduced": {
+// MR-NEXT:       "major": 10,
+// MR-NEXT:       "minor": 0,
+// MR-NEXT:       "patch": 0
+// MR-NEXT:     }
+// MR-NEXT:     "obsoleted": {
+// MR-NEXT:       "major": 19,
+// MR-NEXT:       "minor": 0,
+// MR-NEXT:       "patch": 0
+// MR-NEXT:     }
+// MR-NEXT:   }
+// MR-NEXT: ]
+
+@end
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix B
+__attribute__((deprecated("B is deprecated")))
+@interface B
+// B-LABEL: "!testLabel": "c:objc(cs)B"
+// B:      "availability": [
+// B-NEXT:   {
+// B-NEXT:     "domain": "*"
+// B-NEXT:     "isUnconditionallyDeprecated": true
+// B-NEXT:   }
+// B-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix BIP
+@property int BIP;
+// BIP-LABEL: "!testLabel": "c:objc(cs)B(py)BIP"
+// BIP:      "availability": [
+// BIP-NEXT:   {
+// BIP-NEXT:     "domain": "*"
+// BIP-NEXT:     "isUnconditionallyDeprecated": true
+// BIP-NEXT:   }
+// BIP-NEXT: ]
+@end
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix C
+__attribute__((availability(macos, unavailable)))
+@interface C
+// C-LABEL: "!testLabel": "c:objc(cs)C"
+// C:      "availability": [
+// C-NEXT:   {
+// C-NEXT:     "domain": "macos"
+// C-NEXT:     "isUnconditionallyUnavailable": true
+// C-NEXT:   }
+// C-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix CIP
+@property int CIP;
+// CIP-LABEL: "!testLabel": "c:objc(cs)C(py)CIP"
+// CIP:      "availability": [
+// CIP-NEXT:   {
+// CIP-NEXT:     "domain": "macos"
+// CIP-NEXT:     "isUnconditionallyUnavailable": true
+// CIP-NEXT:   }
+// CIP-NEXT: ]
+@end
+
+@interface D
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix DIP
+@property int DIP __attribute__((availability(macos, introduced=10.0, deprecated=11.0, obsoleted=19.0)));
+// DIP-LABEL: "!testLabel": "c:objc(cs)D(py)DIP"
+// DIP:      "availability": [
+// DIP-NEXT:   {
+// DIP-NEXT:     "deprecated": {
+// DIP-NEXT:       "major": 11,
+// DIP-NEXT:       "minor": 0,
+// DIP-NEXT:       "patch": 0
+// DIP-NEXT:     }
+// DIP-NEXT:     "domain": "macos"
+// DIP-NEXT:     "introduced": {
+// DIP-NEXT:       "major": 10,
+// DIP-NEXT:       "minor": 0,
+// DIP-NEXT:       "patch": 0
+// DIP-NEXT:     }
+// DIP-NEXT:     "obsoleted": {
+// DIP-NEXT:       "major": 19,
+// DIP-NEXT:       "minor": 0,
+// DIP-NEXT:       "patch": 0
+// DIP-NEXT:     }
+// DIP-NEXT:   }
+// DIP-NEXT: ]
+@end
+
+// expected-no-diagnostics
diff --git a/clang/test/ExtractAPI/platform-serialization.c b/clang/test/ExtractAPI/platform-serialization.c
new file mode 100644
index 000000000000000..6d6a13f085cd9d9
--- /dev/null
+++ b/clang/test/ExtractAPI/platform-serialization.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -extract-api --pretty-sgf -triple arm64-apple-ios17.1-macabi \
+// RUN:   -x c-header %s -verify -o - | FileCheck %s
+
+int a;
+
+// CHECK:      "platform": {
+// CHECK-NEXT:   "architecture": "arm64",
+// CHECK-NEXT:   "environment": "macabi",
+// CHECK-NEXT:   "operatingSystem": {
+// CHECK-NEXT:     "minimumVersion": {
+// CHECK-NEXT:       "major": 14,
+// CHECK-NEXT:       "minor": 0,
+// CHECK-NEXT:       "patch": 0
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "name": "ios"
+// CHECK-NEXT:   },
+// CHECK-NEXT:   "vendor": "apple"
+// CHECK-NEXT: }
+
+// expected-no-diagnostics
diff --git a/clang/test/Lexer/cxx-features.cpp b/clang/test/Lexer/cxx-features.cpp
index 08b732132228bac..1c51013ca06f774 100644
--- a/clang/test/Lexer/cxx-features.cpp
+++ b/clang/test/Lexer/cxx-features.cpp
@@ -34,6 +34,10 @@
 
 // --- C++26 features ---
 
+#if check(variadic_friend, 202403, 202403, 202403, 202403, 202403, 202403, 202403)
+#error "wrong value for __cpp_variadic_friend"
+#endif
+
 #if check(deleted_function, 202403, 202403, 202403, 202403, 202403, 202403, 202403)
 #error "wrong value for __cpp_deleted_function"
 #endif
diff --git a/clang/test/Modules/enum-codegen.cpp b/clang/test/Modules/enum-codegen.cpp
index 4397b457801da4e..0028ab00c15290f 100644
--- a/clang/test/Modules/enum-codegen.cpp
+++ b/clang/test/Modules/enum-codegen.cpp
@@ -1,5 +1,6 @@
 // RUN: rm -rf %t
 // RUN: %clang_cc1 -triple %itanium_abi_triple -fmodules -fmodules-cache-path=%t %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -fmodules -fmodules-cache-path=%t %s -emit-llvm -o - -fexperimental-new-constant-interpreter | FileCheck %s
 
 // CHECK: @{{.*var.*}} = {{.*}} %union.union_type { i8 1 },
 
diff --git a/clang/test/Modules/merge-using-decls.cpp b/clang/test/Modules/merge-using-decls.cpp
index e3bf977f0544993..e9794a40837ef97 100644
--- a/clang/test/Modules/merge-using-decls.cpp
+++ b/clang/test/Modules/merge-using-decls.cpp
@@ -6,7 +6,8 @@
 // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify %s -DORDER=2
 // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify -std=c++98 %s -DORDER=2
 // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify -std=c++11 %s -DORDER=2
-// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify -std=c++17 %s -DORDER=2
+
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify -std=c++17 %s -DORDER=2
 
 #if ORDER == 1
 #include "a.h"
diff --git a/clang/test/OpenMP/loop_collapse_1.c b/clang/test/OpenMP/loop_collapse_1.c
new file mode 100644
index 000000000000000..c9877419223dde4
--- /dev/null
+++ b/clang/test/OpenMP/loop_collapse_1.c
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -verify %s
+
+void func( double *A, int N, int M, int NB ) {
+#pragma omp parallel
+  {
+    int nblks = (N-1)/NB;
+    int lnb = ((N-1)/NB)*NB;
+
+#pragma omp for collapse(2)
+    for (int jblk = 0 ; jblk < nblks ; jblk++ ) {
+      int jb = (jblk == nblks - 1 ? lnb : NB);
+      for (int jk = 0; jk < N; jk+=jb) {  // expected-error{{cannot use variable 'jb' in collapsed imperfectly-nested loop increment statement}}
+      }
+    }
+
+#pragma omp for collapse(2)
+    for (int a = 0; a < N; a++) {
+      for (int b = 0; b < M; b++) {
+        int cx = a+b < NB ? a : b;
+        for (int c = 0; c < cx; c++) {
+        }
+      }
+    }
+
+#pragma omp for collapse(3)
+    for (int a = 0; a < N; a++) {
+      for (int b = 0; b < M; b++) {
+        int cx = a+b < NB ? a : b;
+        for (int c = 0; c < cx; c++) {  // expected-error{{cannot use variable 'cx' in collapsed imperfectly-nested loop condition statement}}
+        }
+      }
+    }
+  }
+}
+
+int main(void) {
+  double arr[256];
+  func (arr, 16, 16, 16);
+  return 0;
+}
diff --git a/clang/test/OpenMP/loop_collapse_2.cpp b/clang/test/OpenMP/loop_collapse_2.cpp
new file mode 100644
index 000000000000000..59deddf65e37b54
--- /dev/null
+++ b/clang/test/OpenMP/loop_collapse_2.cpp
@@ -0,0 +1,80 @@
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -verify %s
+
+// We just want to try out a range for statement... this seems a bit OTT.
+template<typename T>
+class fakevector {
+  T *contents;
+  long size;
+public:
+  fakevector(long sz) : size(sz) {
+    contents = new T[sz];
+  }
+  ~fakevector() {
+    delete[] contents;
+  }
+  T& operator[](long x) { return contents[x]; }
+  typedef T *iterator;
+  fakevector<T>::iterator begin() {
+    return &contents[0];
+  }
+  fakevector<T>::iterator end() {
+    return &contents[size];
+  }
+};
+
+void func( double *A, int N, int M, int NB ) {
+#pragma omp parallel
+  {
+    int nblks = (N-1)/NB;
+    int lnb = ((N-1)/NB)*NB;
+#pragma omp for collapse(2)
+    for (int jblk = 0 ; jblk < nblks ; jblk++ ) {
+      int jb = (jblk == nblks - 1 ? lnb : NB);
+      for (int jk = 0; jk < N; jk+=jb) {  // expected-error{{cannot use variable 'jb' in collapsed imperfectly-nested loop increment statement}}
+      }
+    }
+
+#pragma omp for collapse(2)
+    for (int a = 0; a < N; a++) {
+        for (int b = 0; b < M; b++) {
+          int cx = a+b < NB ? a : b;
+          for (int c = 0; c < cx; c++) {
+          }
+        }
+    }
+
+    fakevector<float> myvec{N};
+#pragma omp for collapse(2)
+    for (auto &a : myvec) {
+      fakevector<float> myvec3{M};
+      for (auto &b : myvec3) {  // expected-error{{cannot use variable 'myvec3' in collapsed imperfectly-nested loop init statement}}
+      }
+    }
+
+    fakevector<float> myvec2{M};
+
+#pragma omp for collapse(3)
+    for (auto &a : myvec) {
+      for (auto &b : myvec2) {
+        int cx = a < b ? N : M;
+        for (int c = 0; c < cx; c++) {  // expected-error {{cannot use variable 'cx' in collapsed imperfectly-nested loop condition statement}}
+        }
+      }
+    }
+
+#pragma omp for collapse(3)
+    for (auto &a : myvec) {
+      int cx = a < 5 ? M : N;
+      for (auto &b : myvec2) {
+        for (int c = 0; c < cx; c++) {  // expected-error{{cannot use variable 'cx' in collapsed imperfectly-nested loop condition statement}}
+        }
+      }
+    }
+  }
+}
+
+int main(void) {
+  double arr[256];
+  func (arr, 16, 16, 16);
+  return 0;
+}
diff --git a/clang/test/OpenMP/target_teams_codegen.cpp b/clang/test/OpenMP/target_teams_codegen.cpp
index 9cab8eef1488330..13d44e127201bda 100644
--- a/clang/test/OpenMP/target_teams_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_codegen.cpp
@@ -127,13 +127,13 @@ int foo(int n) {
     aa += 1;
   }
 
-  #pragma omp target teams ompx_bare num_teams(1, 2) thread_limit(1)
+  #pragma omp target teams ompx_bare num_teams(1, 2) thread_limit(1, 2)
   {
     a += 1;
     aa += 1;
   }
 
-  #pragma omp target teams ompx_bare num_teams(1, 2, 3) thread_limit(1)
+  #pragma omp target teams ompx_bare num_teams(1, 2, 3) thread_limit(1, 2, 3)
   {
     a += 1;
     aa += 1;
@@ -667,7 +667,7 @@ int bar(int n){
 // CHECK1-NEXT:    [[TMP144:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 10
 // CHECK1-NEXT:    store [3 x i32] [i32 1, i32 2, i32 0], ptr [[TMP144]], align 4
 // CHECK1-NEXT:    [[TMP145:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 11
-// CHECK1-NEXT:    store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP145]], align 4
+// CHECK1-NEXT:    store [3 x i32] [i32 1, i32 2, i32 0], ptr [[TMP145]], align 4
 // CHECK1-NEXT:    [[TMP146:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 12
 // CHECK1-NEXT:    store i32 0, ptr [[TMP146]], align 4
 // CHECK1-NEXT:    [[TMP147:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l130.region_id, ptr [[KERNEL_ARGS29]])
@@ -720,7 +720,7 @@ int bar(int n){
 // CHECK1-NEXT:    [[TMP171:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 10
 // CHECK1-NEXT:    store [3 x i32] [i32 1, i32 2, i32 3], ptr [[TMP171]], align 4
 // CHECK1-NEXT:    [[TMP172:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 11
-// CHECK1-NEXT:    store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP172]], align 4
+// CHECK1-NEXT:    store [3 x i32] [i32 1, i32 2, i32 3], ptr [[TMP172]], align 4
 // CHECK1-NEXT:    [[TMP173:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 12
 // CHECK1-NEXT:    store i32 0, ptr [[TMP173]], align 4
 // CHECK1-NEXT:    [[TMP174:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l136.region_id, ptr [[KERNEL_ARGS37]])
@@ -2458,7 +2458,7 @@ int bar(int n){
 // CHECK3-NEXT:    [[TMP142:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 10
 // CHECK3-NEXT:    store [3 x i32] [i32 1, i32 2, i32 0], ptr [[TMP142]], align 4
 // CHECK3-NEXT:    [[TMP143:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 11
-// CHECK3-NEXT:    store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP143]], align 4
+// CHECK3-NEXT:    store [3 x i32] [i32 1, i32 2, i32 0], ptr [[TMP143]], align 4
 // CHECK3-NEXT:    [[TMP144:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 12
 // CHECK3-NEXT:    store i32 0, ptr [[TMP144]], align 4
 // CHECK3-NEXT:    [[TMP145:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l130.region_id, ptr [[KERNEL_ARGS29]])
@@ -2511,7 +2511,7 @@ int bar(int n){
 // CHECK3-NEXT:    [[TMP169:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 10
 // CHECK3-NEXT:    store [3 x i32] [i32 1, i32 2, i32 3], ptr [[TMP169]], align 4
 // CHECK3-NEXT:    [[TMP170:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 11
-// CHECK3-NEXT:    store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP170]], align 4
+// CHECK3-NEXT:    store [3 x i32] [i32 1, i32 2, i32 3], ptr [[TMP170]], align 4
 // CHECK3-NEXT:    [[TMP171:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 12
 // CHECK3-NEXT:    store i32 0, ptr [[TMP171]], align 4
 // CHECK3-NEXT:    [[TMP172:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l136.region_id, ptr [[KERNEL_ARGS37]])
diff --git a/clang/test/Parser/attr-order.cpp b/clang/test/Parser/attr-order.cpp
index 10bad38cac64470..21e30e5b6f9df1a 100644
--- a/clang/test/Parser/attr-order.cpp
+++ b/clang/test/Parser/attr-order.cpp
@@ -31,3 +31,16 @@ template <int a>
 
 template <int a>
 [[noreturn]] __declspec(dllexport) __attribute__((cdecl)) void k(); // ok
+
+extern "C" {
+  __attribute__ ((__warn_unused_result__)) [[__maybe_unused__]] int l(int); // ok
+  [[__maybe_unused__]] __attribute__ ((__warn_unused_result__)) int m(int); // ok
+}
+
+extern "C" {
+  __attribute__ ((__warn_unused_result__)) [[__maybe_unused__]] int n (int); // ok
+  __attribute__ ((__warn_unused_result__)) [[__maybe_unused__]] static int o (int x) { return x; }; // ok
+}
+
+extern "C" __attribute__ ((__warn_unused_result__)) [[__maybe_unused__]] int p(int); // ok
+extern "C" [[__maybe_unused__]] __attribute__ ((__warn_unused_result__)) int q(int); // ok
diff --git a/clang/test/Parser/cxx2c-variadic-friends-ext-diags.cpp b/clang/test/Parser/cxx2c-variadic-friends-ext-diags.cpp
new file mode 100644
index 000000000000000..ffcc97ffd635298
--- /dev/null
+++ b/clang/test/Parser/cxx2c-variadic-friends-ext-diags.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -std=c++2c -verify=compat -fsyntax-only -Wpre-c++26-compat %s
+// RUN: %clang_cc1 -std=c++11 -verify=pre2c -fsyntax-only -Wc++26-extensions %s
+
+struct S {
+  friend int, long, char; // compat-warning {{variadic 'friend' declarations are incompatible with C++ standards before C++2c}} \
+                          // pre2c-warning {{variadic 'friend' declarations are a C++2c extension}}
+};
+
+template <typename ...Types>
+struct TS {
+  friend Types...; // compat-warning {{variadic 'friend' declarations are incompatible with C++ standards before C++2c}} \
+                   // pre2c-warning {{variadic 'friend' declarations are a C++2c extension}}
+
+  friend int, Types..., Types...; // compat-warning {{variadic 'friend' declarations are incompatible with C++ standards before C++2c}} \
+                                  // pre2c-warning {{variadic 'friend' declarations are a C++2c extension}}
+};
diff --git a/clang/test/Parser/cxx2c-variadic-friends.cpp b/clang/test/Parser/cxx2c-variadic-friends.cpp
new file mode 100644
index 000000000000000..b7da3e611048129
--- /dev/null
+++ b/clang/test/Parser/cxx2c-variadic-friends.cpp
@@ -0,0 +1,91 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2c %s
+
+template <typename> struct TS; // #template
+
+struct Errors {
+  friend int, int;
+  friend int, long, char;
+
+  // We simply diagnose and ignore the '...' here.
+  friend float...; // expected-error {{pack expansion does not contain any unexpanded parameter packs}}
+
+  friend short..., unsigned, unsigned short...; // expected-error 2 {{pack expansion does not contain any unexpanded parameter packs}}
+
+  template <typename>
+  friend struct TS, int; // expected-error {{a friend declaration that befriends a template must contain exactly one type-specifier}}
+
+  double friend; // expected-error {{'friend' must appear first in a non-function declaration}}
+  double friend, double; // expected-error {{expected member name or ';' after declaration specifiers}}
+};
+
+template <typename>
+struct C { template<class T> class Nested; };
+
+template <typename, typename>
+struct D { template<class T> class Nested; };
+
+template <bool>
+struct E { template<class T> class Nested; };
+
+template<class... Ts> // expected-note {{template parameter is declared here}}
+struct VS {
+  friend Ts...;
+
+  friend class Ts...; // expected-error {{declaration of 'Ts' shadows template parameter}}
+  // expected-error@-1 {{pack expansion does not contain any unexpanded parameter packs}}
+
+  // TODO: Fix-it hint to insert '...'.
+  friend Ts; // expected-error {{friend declaration contains unexpanded parameter pack}}
+
+  template<class... Us>
+  friend Us...; // expected-error {{friend type templates must use an elaborated type}}
+
+  template<class... Us> // expected-note {{is declared here}}
+  friend class Us...; // expected-error {{declaration of 'Us' shadows template parameter}}
+
+  template<class U>
+  friend class C<Ts>::template Nested<U>...; // expected-error {{cannot specialize a dependent template}}
+
+  template<class... Us>
+  friend class C<Ts...>::template Nested<Us>...; // expected-error {{cannot specialize a dependent template}}
+
+  // Nonsense (see CWG 2917).
+  template<class... Us>
+  friend class C<Us>::Nested...; // expected-error {{friend declaration expands pack 'Us' that is declared it its own template parameter list}}
+
+  template<bool... Bs>
+  friend class E<Bs>::Nested...; // expected-error {{friend declaration expands pack 'Bs' that is declared it its own template parameter list}}
+
+  // FIXME: Both of these should be valid, but we can't handle these at
+  // the moment because the NNS is dependent.
+  template<class ...T>
+  friend class TS<Ts>::Nested...; // expected-warning {{dependent nested name specifier 'TS<Ts>::' for friend template declaration is not supported; ignoring this friend declaration}}
+
+  template<class T>
+  friend class D<T, Ts>::Nested...; // expected-warning {{dependent nested name specifier 'D<T, Ts>::' for friend class declaration is not supported; turning off access control for 'VS'}}
+};
+
+namespace length_mismatch {
+struct A {
+  template <typename...>
+  struct Nested {
+    struct Foo{};
+  };
+};
+template <typename ...Ts>
+struct S {
+  template <typename ...Us>
+  struct T {
+    // expected-error@+2 {{pack expansion contains parameter packs 'Ts' and 'Us' that have different lengths (1 vs. 2)}}
+    // expected-error@+1 {{pack expansion contains parameter packs 'Ts' and 'Us' that have different lengths (2 vs. 1)}}
+    friend class Ts::template Nested<Us>::Foo...;
+  };
+};
+
+void f() {
+  S<A>::T<int> s;
+  S<A, A>::T<int, long> s2;
+  S<A>::T<int, long> s3; // expected-note {{in instantiation of}}
+  S<A, A>::T<int> s4; // expected-note {{in instantiation of}}
+}
+}
diff --git a/clang/test/Parser/parser_overflow.c b/clang/test/Parser/parser_overflow.c
index 9514e808550a4b8..53c79bc06d993d9 100644
--- a/clang/test/Parser/parser_overflow.c
+++ b/clang/test/Parser/parser_overflow.c
@@ -1,5 +1,5 @@
 // RUN: not %clang_cc1 %s -fsyntax-only -DHUGE 2>&1 | FileCheck %s
-// RUN: not %clang_cc1 %s -fsyntax-only 2>&1 | FileCheck %s
+// RUN: %clang_cc1 %s -fsyntax-only
 // RUN: not %clang_cc1 %s -fsyntax-only -fbracket-depth 299 2>&1 | FileCheck %s
 // RUN: %clang_cc1 %s -fsyntax-only -fbracket-depth 300
 // RUN: not %clang %s -fsyntax-only -fbracket-depth=299 2>&1 | FileCheck %s
@@ -15,5 +15,5 @@ void foo(void) {
 #endif
 }
 
-// CHECK: fatal error: bracket nesting level exceeded maximum of {{256|299}}
+// CHECK: fatal error: bracket nesting level exceeded maximum of {{2048|299}}
 // CHECK: note: use -fbracket-depth=N to increase maximum nesting level
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 72131108cb5f6ac..517702fab5b919f 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -187,8 +187,10 @@
 // CHECK-NOT: __riscv_zicfilp {{.*$}}
 // CHECK-NOT: __riscv_zicfiss {{.*$}}
 // CHECK-NOT: __riscv_ztso {{.*$}}
+// CHECK-NOT: __riscv_zvbc32e {{.*$}}
 // CHECK-NOT: __riscv_zvfbfmin {{.*$}}
 // CHECK-NOT: __riscv_zvfbfwma {{.*$}}
+// CHECK-NOT: __riscv_zvkgs {{.*$}}
 
 // RUN: %clang --target=riscv32-unknown-linux-gnu \
 // RUN:   -march=rv32ia -E -dM %s \
@@ -1658,6 +1660,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZTSO-EXT %s
 // CHECK-ZTSO-EXT: __riscv_ztso 1000000{{$}}
 
+// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN:   -march=rv32i_zve32x_zvbc32e0p7 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-ZVBC32E-EXT %s
+// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN:   -march=rv64i_zve32x_zvbc32e0p7 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-ZVBC32E-EXT %s
+// CHECK-ZVBC32E-EXT: __riscv_zvbc32e 7000{{$}}
+
 // RUN: %clang --target=riscv32 -menable-experimental-extensions \
 // RUN:   -march=rv32ifzvfbfmin1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZVFBFMIN-EXT %s
@@ -1674,6 +1684,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZVFBFWMA-EXT %s
 // CHECK-ZVFBFWMA-EXT: __riscv_zvfbfwma 1000000{{$}}
 
+// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN:   -march=rv32i_zve32x_zvkgs0p7 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-ZVKGS-EXT %s
+// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN:   -march=rv64i_zve32x_zvkgs0p7 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-ZVKGS-EXT %s
+// CHECK-ZVKGS-EXT: __riscv_zvkgs 7000{{$}}
+
 // RUN: %clang -target riscv32 -menable-experimental-extensions \
 // RUN:   -march=rv32izicfiss1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZICFISS-EXT %s
diff --git a/clang/test/Sema/block-misc.c b/clang/test/Sema/block-misc.c
index aea44d55a606a41..c8a34b7f3c9fd55 100644
--- a/clang/test/Sema/block-misc.c
+++ b/clang/test/Sema/block-misc.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -Wno-strict-prototypes -verify %s -fblocks
+// RUN: %clang_cc1 -fsyntax-only -Wno-strict-prototypes -verify %s -fblocks -fexperimental-new-constant-interpreter
 void donotwarn(void);
 
 int (^IFP) ();
diff --git a/clang/test/Sema/block-return.c b/clang/test/Sema/block-return.c
index d3d70511b185138..126fc6f953dea2e 100644
--- a/clang/test/Sema/block-return.c
+++ b/clang/test/Sema/block-return.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -Wno-int-to-pointer-cast -Wno-pointer-to-int-cast -pedantic -fsyntax-only %s -verify -fblocks
+// RUN: %clang_cc1 -Wno-int-to-pointer-cast -Wno-pointer-to-int-cast -pedantic -fsyntax-only %s -verify -fblocks -fexperimental-new-constant-interpreter
 
 extern int printf(const char *, ...);
 
diff --git a/clang/test/Sema/c2x-nodiscard.c b/clang/test/Sema/c2x-nodiscard.c
index cb33c0c242e7db9..f8b0567366465df 100644
--- a/clang/test/Sema/c2x-nodiscard.c
+++ b/clang/test/Sema/c2x-nodiscard.c
@@ -54,3 +54,9 @@ void test_missiles(void) {
   launch_missiles();
 }
 
+[[nodiscard]] int f3();
+
+void GH104391() {
+#define M (unsigned int) f3()
+  M; // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+}
diff --git a/clang/test/Sema/constant-builtins-2.c b/clang/test/Sema/constant-builtins-2.c
index 37b63cf4f6b3285..da2264500d76802 100644
--- a/clang/test/Sema/constant-builtins-2.c
+++ b/clang/test/Sema/constant-builtins-2.c
@@ -204,6 +204,53 @@ char isfpclass_snan_1   [!__builtin_isfpclass(__builtin_nans(""), 0x0002) ? 1 :
 char isfpclass_snan_2   [__builtin_isfpclass(__builtin_nansl(""), 0x0207) ? 1 : -1]; // ~fcFinite
 char isfpclass_snan_3   [!__builtin_isfpclass(__builtin_nans(""), 0x01F8) ? 1 : -1]; // fcFinite
 
+__extension__ _Static_assert(
+  !__builtin_signbit(1.0) && __builtin_signbit(-1.0) && !__builtin_signbit(0.0) && __builtin_signbit(-0.0) &&
+  !__builtin_signbitf(1.0f) && __builtin_signbitf(-1.0f) && !__builtin_signbitf(0.0f) && __builtin_signbitf(-0.0f) &&
+  !__builtin_signbitl(1.0L) && __builtin_signbitf(-1.0L) && !__builtin_signbitf(0.0L) && __builtin_signbitf(-0.0L) &&
+  !__builtin_signbit(1.0f) && __builtin_signbit(-1.0f) && !__builtin_signbit(0.0f) && __builtin_signbit(-0.0f) &&
+  !__builtin_signbit(1.0L) && __builtin_signbit(-1.0L) && !__builtin_signbit(0.0L) && __builtin_signbit(-0.0L) &&
+#if defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__)
+  !__builtin_signbit(1.0q) && __builtin_signbit(-1.0q) && !__builtin_signbit(0.0q) && __builtin_signbit(-0.0q) &&
+#endif
+  1, ""
+);
+
+#define LESS(X, Y) \
+  !__builtin_isgreater(X, Y) && __builtin_isgreater(Y, X) &&             \
+  !__builtin_isgreaterequal(X, Y) && __builtin_isgreaterequal(Y, X) &&   \
+  __builtin_isless(X, Y) && !__builtin_isless(Y, X) &&                   \
+  __builtin_islessequal(X, Y) && !__builtin_islessequal(Y, X) &&         \
+  __builtin_islessgreater(X, Y) && __builtin_islessgreater(Y, X) &&      \
+  !__builtin_isunordered(X, Y) && !__builtin_isunordered(Y, X)
+#define EQUAL(X, Y) \
+  !__builtin_isgreater(X, Y) && !__builtin_isgreater(Y, X) &&            \
+  __builtin_isgreaterequal(X, Y) && __builtin_isgreaterequal(Y, X) &&    \
+  !__builtin_isless(X, Y) && !__builtin_isless(Y, X) &&                  \
+  __builtin_islessequal(X, Y) && __builtin_islessequal(Y, X) &&          \
+  !__builtin_islessgreater(X, Y) && !__builtin_islessgreater(Y, X) &&    \
+  !__builtin_isunordered(X, Y) && !__builtin_isunordered(Y, X)
+#define UNORDERED(X, Y) \
+  !__builtin_isgreater(X, Y) && !__builtin_isgreater(Y, X) &&            \
+  !__builtin_isgreaterequal(X, Y) && !__builtin_isgreaterequal(Y, X) &&  \
+  !__builtin_isless(X, Y) && !__builtin_isless(Y, X) &&                  \
+  !__builtin_islessequal(X, Y) && !__builtin_islessequal(Y, X) &&        \
+  !__builtin_islessgreater(X, Y) && !__builtin_islessgreater(Y, X) &&    \
+  __builtin_isunordered(X, Y) && __builtin_isunordered(Y, X)
+
+__extension__ _Static_assert(
+  LESS(0.0, 1.0) && EQUAL(1.0, 1.0) && EQUAL(0.0, -0.0) &&
+  UNORDERED(__builtin_nan(""), 1.0) && UNORDERED(__builtin_nan(""), __builtin_inf()) && LESS(0.0, __builtin_inf()) &&
+  LESS(0.0f, 1.0f) && EQUAL(1.0f, 1.0f) && EQUAL(0.0f, -0.0f) &&
+  UNORDERED(__builtin_nanf(""), 1.0f) && UNORDERED(__builtin_nanf(""), __builtin_inff()) && LESS(0.0f, __builtin_inff()) &&
+  LESS(0.0L, 1.0L) && EQUAL(1.0L, 1.0L) && EQUAL(0.0L, -0.0L) &&
+  UNORDERED(__builtin_nanl(""), 1.0L) && UNORDERED(__builtin_nanl(""), __builtin_infl()) && LESS(0.0L, __builtin_infl()) &&
+#if defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__)
+  LESS(0.0q, 1.0q) && EQUAL(1.0q, 1.0q) && EQUAL(0.0q, -0.0q) &&
+#endif
+  1, ""
+);
+
 //double       g19 = __builtin_powi(2.0, 4);
 //float        g20 = __builtin_powif(2.0f, 4);
 //long double  g21 = __builtin_powil(2.0L, 4);
diff --git a/clang/test/SemaCXX/MicrosoftExtensions.cpp b/clang/test/SemaCXX/MicrosoftExtensions.cpp
index 98c19975095bbee..7454a01158f6b4a 100644
--- a/clang/test/SemaCXX/MicrosoftExtensions.cpp
+++ b/clang/test/SemaCXX/MicrosoftExtensions.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -std=c++17 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
-// RUN: %clang_cc1 -std=c++98 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17 -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
-// RUN: %clang_cc1 -std=c++11 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17 -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
-// RUN: %clang_cc1 -std=c++14 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17 -fexceptions -fcxx-exceptions -DTEST2
-// RUN: %clang_cc1 %s -triple i686-pc-win32 -fsyntax-only -std=c++11 -fms-compatibility -verify -DTEST3
+// RUN: %clang_cc1 -std=c++17 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,ms-union-ext -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
+// RUN: %clang_cc1 -std=c++98 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17,ms-union-ext -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
+// RUN: %clang_cc1 -std=c++11 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17,ms-union-ext -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
+// RUN: %clang_cc1 -std=c++14 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17,ms-union-ext-disabled -fexceptions -fcxx-exceptions -DTEST2
+// RUN: %clang_cc1 %s -triple i686-pc-win32 -fsyntax-only -std=c++11 -fms-compatibility -verify=expected,ms-union-ext -DTEST3
+// RUN: %clang_cc1 -std=c++17 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -verify=ms-union-ext -fms-extensions -fms-compatibility-version=18.00
+// RUN: %clang_cc1 -std=c++17 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -verify=ms-union-ext-disabled -fms-extensions -fms-compatibility-version=19.00
 
 #if TEST1
 
@@ -384,11 +386,6 @@ void TestSP9() {
   c3.h(); // Overloaded unary op operand
 }
 
-union u {
-  int *i1;
-  int &i2;  // expected-warning {{union member 'i2' has reference type 'int &', which is a Microsoft extension}}
-};
-
 // Property getter using reference.
 struct SP11 {
   __declspec(property(get=GetV)) int V;
@@ -619,9 +616,12 @@ template<typename T> struct A {};
 template<typename T> struct B : A<A<T>> { A<T>::C::D d; }; // expected-warning {{implicit 'typename' is a C++20 extension}}
 }
 
-#else
-
-#error Unknown test mode
-
 #endif
 
+union u {
+    int *i1;
+
+    // ms-union-ext-warning@+2 {{union member 'i2' has reference type 'int &', which is a Microsoft extension}}
+    // ms-union-ext-disabled-error@+1 {{union member 'i2' has reference type 'int &'}}
+    int &i2;
+};
diff --git a/clang/test/SemaCXX/attr-lifetimebound.cpp b/clang/test/SemaCXX/attr-lifetimebound.cpp
index 7db0a4d64d2596c..f4ca840cb276f9f 100644
--- a/clang/test/SemaCXX/attr-lifetimebound.cpp
+++ b/clang/test/SemaCXX/attr-lifetimebound.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2a -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
 
 namespace usage_invalid {
   // FIXME: Should we diagnose a void return type?
@@ -9,6 +9,7 @@ namespace usage_invalid {
     A() [[clang::lifetimebound]]; // expected-error {{cannot be applied to a constructor}}
     ~A() [[clang::lifetimebound]]; // expected-error {{cannot be applied to a destructor}}
     static int *static_class_member() [[clang::lifetimebound]]; // expected-error {{static member function has no implicit object parameter}}
+    int *explicit_object(this A&) [[clang::lifetimebound]]; // expected-error {{explicit object member function has no implicit object parameter}}
     int not_function [[clang::lifetimebound]]; // expected-error {{only applies to parameters and implicit object parameters}}
     int [[clang::lifetimebound]] also_not_function; // expected-error {{cannot be applied to types}}
   };
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index d888887bd8c6f39..44ef540f41fa8c7 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -1268,6 +1268,9 @@ constexpr complex_wrap makeComplexWrap(int re, int im) {
 static_assert(makeComplexWrap(1,0) == complex(1), "");
 static_assert(makeComplexWrap(1,0) != complex(0, 1), "");
 
+constexpr auto GH55390 = 1 / 65536j; // expected-note {{division by zero}} \
+                                     // expected-error {{constexpr variable 'GH55390' must be initialized by a constant expression}} \
+                                     // expected-warning {{imaginary constants are a GNU extension}}
 }
 
 namespace PR11595 {
diff --git a/clang/test/SemaCXX/consteval-cleanup.cpp b/clang/test/SemaCXX/consteval-cleanup.cpp
index 499c45db5017702..f7d033b2ecafa46 100644
--- a/clang/test/SemaCXX/consteval-cleanup.cpp
+++ b/clang/test/SemaCXX/consteval-cleanup.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fblocks -Wno-unused-value -std=c++20 -ast-dump -verify %s -ast-dump | FileCheck %s
+// RUN: %clang_cc1 -fblocks -Wno-unused-value -std=c++20 -ast-dump -verify %s -ast-dump -fexperimental-new-constant-interpreter | FileCheck %s
 
 // expected-no-diagnostics
 
diff --git a/clang/test/SemaCXX/constexpr-vectors.cpp b/clang/test/SemaCXX/constexpr-vectors.cpp
index 99b045f888d87c2..7492116f44d70f3 100644
--- a/clang/test/SemaCXX/constexpr-vectors.cpp
+++ b/clang/test/SemaCXX/constexpr-vectors.cpp
@@ -1,10 +1,6 @@
-// RUN: %clang_cc1 -std=c++14 -Wno-unused-value %s -disable-llvm-passes -triple x86_64-linux-gnu -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple x86_64-linux-gnu -Wno-uninitialized -std=c++14 -fsyntax-only -verify
 
-// FIXME: Unfortunately there is no good way to validate that our values are
-// correct since Vector types don't have operator [] implemented for constexpr.
-// Instead, we need to use filecheck to ensure the emitted IR is correct. Once
-// someone implements array subscript operator for these types as constexpr,
-// this test should modified to jsut use static asserts.
+// expected-no-diagnostics
 
 using FourCharsVecSize __attribute__((vector_size(4))) = char;
 using FourIntsVecSize __attribute__((vector_size(16))) = int;
@@ -150,564 +146,691 @@ constexpr auto CmpBinOr(T t, U u) {
   return t;
 }
 
+constexpr auto CmpF(float t, float u) {
+  return __builtin_fabs(t - u) < 0.0001;
+}
+
 // Only int vs float makes a difference here, so we only need to test 1 of each.
 // Test Char to make sure the mixed-nature of shifts around char is evident.
 void CharUsage() {
   constexpr auto a = FourCharsVecSize{6, 3, 2, 1} +
-                     FourCharsVecSize{12, 15, 5, 7};
-  // CHECK: store <4 x i8> <i8 18, i8 18, i8 7, i8 8>
+            FourCharsVecSize{12, 15, 5, 7};
+  static_assert(a[0] == 18 && a[1] == 18 && a[2] == 7 && a[3] == 8, "");
+
   constexpr auto b = FourCharsVecSize{19, 15, 13, 12} -
                      FourCharsVecSize{13, 14, 5, 3};
-  // CHECK: store <4 x i8> <i8 6, i8 1, i8 8, i8 9>
+  static_assert(b[0] == 6 && b[1] == 1 && b[2] == 8 && b[3] == 9, "");
+
   constexpr auto c = FourCharsVecSize{8, 4, 2, 1} *
                      FourCharsVecSize{3, 4, 5, 6};
-  // CHECK: store <4 x i8> <i8 24, i8 16, i8 10, i8 6>
+  static_assert(c[0] == 24 && c[1] == 16 && c[2] == 10 && c[3] == 6, "");
+
   constexpr auto d = FourCharsVecSize{12, 12, 10, 10} /
                      FourCharsVecSize{6, 4, 5, 2};
-  // CHECK: store <4 x i8> <i8 2, i8 3, i8 2, i8 5>
+  static_assert(d[0] == 2 && d[1] == 3 && d[2] == 2 && d[3] == 5, "");
+
   constexpr auto e = FourCharsVecSize{12, 12, 10, 10} %
                      FourCharsVecSize{6, 4, 4, 3};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 2, i8 1>
+  static_assert(e[0] == 0 && e[1] == 0 && e[2] == 2 && e[3] == 1, "");
 
   constexpr auto f = FourCharsVecSize{6, 3, 2, 1} + 3;
-  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  static_assert(f[0] == 9 && f[1] == 6 && f[2] == 5 && f[3] == 4, "");
+
   constexpr auto g = FourCharsVecSize{19, 15, 12, 10} - 3;
-  // CHECK: store <4 x i8> <i8 16, i8 12, i8 9, i8 7>
+  static_assert(g[0] == 16 && g[1] == 12 && g[2] == 9 && g[3] == 7, "");
+
   constexpr auto h = FourCharsVecSize{8, 4, 2, 1} * 3;
-  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  static_assert(h[0] == 24 && h[1] == 12 && h[2] == 6 && h[3] == 3, "");
+
   constexpr auto j = FourCharsVecSize{12, 15, 18, 21} / 3;
-  // CHECK: store <4 x i8> <i8 4, i8 5, i8 6, i8 7>
+  static_assert(j[0] == 4 && j[1] == 5 && j[2] == 6 && j[3] == 7, "");
+
   constexpr auto k = FourCharsVecSize{12, 17, 19, 22} % 3;
-  // CHECK: store <4 x i8> <i8 0, i8 2, i8 1, i8 1>
+  static_assert(k[0] == 0 && k[1] == 2 && k[2] == 1 && k[3] == 1, "");
 
   constexpr auto l = 3 + FourCharsVecSize{6, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  static_assert(l[0] == 9 && l[1] == 6 && l[2] == 5 && l[3] == 4, "");
+
   constexpr auto m = 20 - FourCharsVecSize{19, 15, 12, 10};
-  // CHECK: store <4 x i8> <i8 1, i8 5, i8 8, i8 10>
+  static_assert(m[0] == 1 && m[1] == 5 && m[2] == 8 && m[3] == 10, "");
+
   constexpr auto n = 3 * FourCharsVecSize{8, 4, 2, 1};
-  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  static_assert(n[0] == 24 && n[1] == 12 && n[2] == 6 && n[3] == 3, "");
+
   constexpr auto o = 100 / FourCharsVecSize{12, 15, 18, 21};
-  // CHECK: store <4 x i8> <i8 8, i8 6, i8 5, i8 4>
+  static_assert(o[0] == 8 && o[1] == 6 && o[2] == 5 && o[3] == 4, "");
+
   constexpr auto p = 100 % FourCharsVecSize{12, 15, 18, 21};
-  // CHECK: store <4 x i8> <i8 4, i8 10, i8 10, i8 16>
+  static_assert(p[0] == 4 && p[1] == 10 && p[2] == 10 && p[3] == 16, "");
 
   constexpr auto q = FourCharsVecSize{6, 3, 2, 1} << FourCharsVecSize{1, 1, 2, 2};
-  // CHECK: store <4 x i8> <i8 12, i8 6, i8 8, i8 4>
+  static_assert(q[0] == 12 && q[1] == 6 && q[2] == 8 && q[3] == 4, "");
+
   constexpr auto r = FourCharsVecSize{19, 15, 12, 10} >>
                      FourCharsVecSize{1, 1, 2, 2};
-  // CHECK: store <4 x i8> <i8 9, i8 7, i8 3, i8 2>
+  static_assert(r[0] == 9 && r[1] == 7 && r[2] == 3 && r[3] == 2, "");
+
   constexpr auto s = FourCharsVecSize{6, 3, 5, 10} << 1;
-  // CHECK: store <4 x i8> <i8 12, i8 6, i8 10, i8 20>
+  static_assert(s[0] == 12 && s[1] == 6 && s[2] == 10 && s[3] == 20, "");
+
   constexpr auto t = FourCharsVecSize{19, 15, 10, 20} >> 1;
-  // CHECK: store <4 x i8> <i8 9, i8 7, i8 5, i8 10>
+  static_assert(t[0] == 9 && t[1] == 7 && t[2] == 5 && t[3] == 10, "");
+
   constexpr auto u = 12 << FourCharsVecSize{1, 2, 3, 3};
-  // CHECK: store <4 x i8> <i8 24, i8 48, i8 96, i8 96>
+  static_assert(u[0] == 24 && u[1] == 48 && u[2] == 96 && u[3] == 96, "");
+
   constexpr auto v = 12 >> FourCharsVecSize{1, 2, 2, 1};
-  // CHECK: store <4 x i8> <i8 6, i8 3, i8 3, i8 6>
+  static_assert(v[0] == 6 && v[1] == 3 && v[2] == 3 && v[3] == 6, "");
 
   constexpr auto w = FourCharsVecSize{1, 2, 3, 4} <
                      FourCharsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 0>
+  static_assert(w[0] == -1 && w[1] == -1 && w[2] == 0 && w[3] == 0, "");
+
   constexpr auto x = FourCharsVecSize{1, 2, 3, 4} >
                      FourCharsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(x[0] == 0 && x[1] == 0 && x[2] == -1 && x[3] == -1, "");
+
   constexpr auto y = FourCharsVecSize{1, 2, 3, 4} <=
                      FourCharsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 -1, i8 0>
+  static_assert(y[0] == -1 && y[1] == -1 && y[2] == -1 && y[3] == 0, "");
+
   constexpr auto z = FourCharsVecSize{1, 2, 3, 4} >=
                      FourCharsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(z[0] == 0 && z[1] == 0 && z[2] == -1 && z[3] == -1, "");
+
   constexpr auto A = FourCharsVecSize{1, 2, 3, 4} ==
                      FourCharsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 0>
+  static_assert(A[0] == 0 && A[1] == 0 && A[2] == -1 && A[3] == 0, "");
+
   constexpr auto B = FourCharsVecSize{1, 2, 3, 4} !=
                      FourCharsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 -1>
+  static_assert(B[0] == -1 && B[1] == -1 && B[2] == 0 && B[3] == -1, "");
 
   constexpr auto C = FourCharsVecSize{1, 2, 3, 4} < 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 0>
+  static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
+
   constexpr auto D = FourCharsVecSize{1, 2, 3, 4} > 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 0, i8 -1>
+  static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
+
   constexpr auto E = FourCharsVecSize{1, 2, 3, 4} <= 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 -1, i8 0>
+  static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
+
   constexpr auto F = FourCharsVecSize{1, 2, 3, 4} >= 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
+
   constexpr auto G = FourCharsVecSize{1, 2, 3, 4} == 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 0>
+  static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
+
   constexpr auto H = FourCharsVecSize{1, 2, 3, 4} != 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 -1>
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");
 
   constexpr auto I = FourCharsVecSize{1, 2, 3, 4} &
                      FourCharsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 2, i8 2, i8 0>
+  static_assert(I[0] == 0 && I[1] == 2 && I[2] == 2 && I[3] == 0, "");
+
   constexpr auto J = FourCharsVecSize{1, 2, 3, 4} ^
                      FourCharsVecSize { 4, 3, 2, 1 };
-  // CHECK: store <4 x i8> <i8 5, i8 1, i8 1, i8 5>
+  static_assert(J[0] == 5 && J[1] == 1 && J[2] == 1 && J[3] == 5, "");
+
   constexpr auto K = FourCharsVecSize{1, 2, 3, 4} |
                      FourCharsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 5, i8 3, i8 3, i8 5>
+  static_assert(K[0] == 5 && K[1] == 3 && K[2] == 3 && K[3] == 5, "");
+
   constexpr auto L = FourCharsVecSize{1, 2, 3, 4} & 3;
-  // CHECK: store <4 x i8> <i8 1, i8 2, i8 3, i8 0>
+  static_assert(L[0] == 1 && L[1] == 2 && L[2] == 3 && L[3] == 0, "");
+
   constexpr auto M = FourCharsVecSize{1, 2, 3, 4} ^ 3;
-  // CHECK: store <4 x i8> <i8 2, i8 1, i8 0, i8 7>
+  static_assert(M[0] == 2 && M[1] == 1 && M[2] == 0 && M[3] == 7, "");
+
   constexpr auto N = FourCharsVecSize{1, 2, 3, 4} | 3;
-  // CHECK: store <4 x i8> <i8 3, i8 3, i8 3, i8 7>
+  static_assert(N[0] == 3 && N[1] == 3 && N[2] == 3 && N[3] == 7, "");
 
   constexpr auto O = FourCharsVecSize{5, 0, 6, 0} &&
                      FourCharsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i8> <i8 1, i8 0, i8 0, i8 0>
+  static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
+
   constexpr auto P = FourCharsVecSize{5, 0, 6, 0} ||
                      FourCharsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+  static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
 
   constexpr auto Q = FourCharsVecSize{5, 0, 6, 0} && 3;
-  // CHECK: store <4 x i8> <i8 1, i8 0, i8 1, i8 0>
+  static_assert(Q[0] == 1 && Q[1] == 0 && Q[2] == 1 && Q[3] == 0, "");
+
   constexpr auto R = FourCharsVecSize{5, 0, 6, 0} || 3;
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 1>
+  static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  // CHECK: store <4 x i8> <i8 108, i8 18, i8 56, i8 72>
+  static_assert(T[0] == 108 && T[1] == 18 && T[2] == 56 && T[3] == 72, "");
 
   constexpr auto U = CmpDiv(a, b);
-  // CHECK: store <4 x i8> <i8 3, i8 18, i8 0, i8 0>
+  static_assert(U[0] == 3 && U[1] == 18 && U[2] == 0 && U[3] == 0, "");
 
   constexpr auto V = CmpRem(a, b);
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 7, i8 8>
+  static_assert(V[0] == 0 && V[1] == 0 && V[2] == 7 && V[3] == 8, "");
 
   constexpr auto X = CmpAdd(a, b);
-  // CHECK: store <4 x i8> <i8 24, i8 19, i8 15, i8 17>
+  static_assert(X[0] == 24 && X[1] == 19 && X[2] == 15 && X[3] == 17, "");
 
   constexpr auto Y = CmpSub(a, b);
-  // CHECK: store <4 x i8> <i8 12, i8 17, i8 -1, i8 -1>
+  static_assert(Y[0] == 12 && Y[1] == 17 && Y[2] == -1 && Y[3] == -1, "");
 
   constexpr auto InvH = -H;
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 1>
+  static_assert(InvH[0] == 1 && InvH[1] == 1 && InvH[2] == 0 && InvH[3] == 1, "");
+
   constexpr auto Z = CmpLSH(a, InvH);
-  // CHECK: store <4 x i8> <i8 36, i8 36, i8 7, i8 16>
+  static_assert(Z[0] == 36 && Z[1] == 36 && Z[2] == 7 && Z[3] == 16, "");
 
   constexpr auto aa = CmpRSH(a, InvH);
-  // CHECK: store <4 x i8> <i8 9, i8 9, i8 7, i8 4>
+  static_assert(aa[0] == 9 && aa[1] == 9 && aa[2] == 7 && aa[3] == 4, "");
 
   constexpr auto ab = CmpBinAnd(a, b);
-  // CHECK: store <4 x i8> <i8 2, i8 0, i8 0, i8 8>
+  static_assert(ab[0] == 2 && ab[1] == 0 && ab[2] == 0 && ab[3] == 8, "");
 
   constexpr auto ac = CmpBinXOr(a, b);
-  // CHECK: store <4 x i8> <i8 20, i8 19, i8 15, i8 1>
+  static_assert(ac[0] == 20 && ac[1] == 19 && ac[2] == 15 && ac[3] == 1, "");
 
   constexpr auto ad = CmpBinOr(a, b);
-  // CHECK: store <4 x i8> <i8 22, i8 19, i8 15, i8 9>
+  static_assert(ad[0] == 22 && ad[1] == 19 && ad[2] == 15 && ad[3] == 9, "");
 
   constexpr auto ae = ~FourCharsVecSize{1, 2, 10, 20};
-  // CHECK: store <4 x i8> <i8 -2, i8 -3, i8 -11, i8 -21>
+  static_assert(ae[0] == -2 && ae[1] == -3 && ae[2] == -11 && ae[3] == -21, "");
 
   constexpr auto af = !FourCharsVecSize{0, 1, 8, -1};
-  // CHECK: store <4 x i8> <i8 -1, i8 0, i8 0, i8 0>
+  static_assert(af[0] == -1 && af[1] == 0 && af[2] == 0 && af[3] == 0, "");
 }
 
 void CharExtVecUsage() {
   constexpr auto a = FourCharsExtVec{6, 3, 2, 1} +
                      FourCharsExtVec{12, 15, 5, 7};
-  // CHECK: store <4 x i8> <i8 18, i8 18, i8 7, i8 8>
+  static_assert(a[0] == 18 && a[1] == 18 && a[2] == 7 && a[3] == 8, "");
+
   constexpr auto b = FourCharsExtVec{19, 15, 13, 12} -
                      FourCharsExtVec{13, 14, 5, 3};
-  // CHECK: store <4 x i8> <i8 6, i8 1, i8 8, i8 9>
+  static_assert(b[0] == 6 && b[1] == 1 && b[2] == 8 && b[3] == 9, "");
+
   constexpr auto c = FourCharsExtVec{8, 4, 2, 1} *
                      FourCharsExtVec{3, 4, 5, 6};
-  // CHECK: store <4 x i8> <i8 24, i8 16, i8 10, i8 6>
+  static_assert(c[0] == 24 && c[1] == 16 && c[2] == 10 && c[3] == 6, "");
+
   constexpr auto d = FourCharsExtVec{12, 12, 10, 10} /
                      FourCharsExtVec{6, 4, 5, 2};
-  // CHECK: store <4 x i8> <i8 2, i8 3, i8 2, i8 5>
+  static_assert(d[0] == 2 && d[1] == 3 && d[2] == 2 && d[3] == 5, "");
+
   constexpr auto e = FourCharsExtVec{12, 12, 10, 10} %
                      FourCharsExtVec{6, 4, 4, 3};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 2, i8 1>
+  static_assert(e[0] == 0 && e[1] == 0 && e[2] == 2 && e[3] == 1, "");
 
   constexpr auto f = FourCharsExtVec{6, 3, 2, 1} + 3;
-  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  static_assert(f[0] == 9 && f[1] == 6 && f[2] == 5 && f[3] == 4, "");
+
   constexpr auto g = FourCharsExtVec{19, 15, 12, 10} - 3;
-  // CHECK: store <4 x i8> <i8 16, i8 12, i8 9, i8 7>
+  static_assert(g[0] == 16 && g[1] == 12 && g[2] == 9 && g[3] == 7, "");
+
   constexpr auto h = FourCharsExtVec{8, 4, 2, 1} * 3;
-  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  static_assert(h[0] == 24 && h[1] == 12 && h[2] == 6 && h[3] == 3, "");
+
   constexpr auto j = FourCharsExtVec{12, 15, 18, 21} / 3;
-  // CHECK: store <4 x i8> <i8 4, i8 5, i8 6, i8 7>
+  static_assert(j[0] == 4 && j[1] == 5 && j[2] == 6 && j[3] == 7, "");
+
   constexpr auto k = FourCharsExtVec{12, 17, 19, 22} % 3;
-  // CHECK: store <4 x i8> <i8 0, i8 2, i8 1, i8 1>
+  static_assert(k[0] == 0 && k[1] == 2 && k[2] == 1 && k[3] == 1, "");
 
   constexpr auto l = 3 + FourCharsExtVec{6, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  static_assert(l[0] == 9 && l[1] == 6 && l[2] == 5 && l[3] == 4, "");
+
   constexpr auto m = 20 - FourCharsExtVec{19, 15, 12, 10};
-  // CHECK: store <4 x i8> <i8 1, i8 5, i8 8, i8 10>
+  static_assert(m[0] == 1 && m[1] == 5 && m[2] == 8 && m[3] == 10, "");
+
   constexpr auto n = 3 * FourCharsExtVec{8, 4, 2, 1};
-  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  static_assert(n[0] == 24 && n[1] == 12 && n[2] == 6 && n[3] == 3, "");
+
   constexpr auto o = 100 / FourCharsExtVec{12, 15, 18, 21};
-  // CHECK: store <4 x i8> <i8 8, i8 6, i8 5, i8 4>
+  static_assert(o[0] == 8 && o[1] == 6 && o[2] == 5 && o[3] == 4, "");
+
   constexpr auto p = 100 % FourCharsExtVec{12, 15, 18, 21};
-  // CHECK: store <4 x i8> <i8 4, i8 10, i8 10, i8 16>
+  static_assert(p[0] == 4 && p[1] == 10 && p[2] == 10 && p[3] == 16, "");
 
   constexpr auto q = FourCharsExtVec{6, 3, 2, 1} << FourCharsVecSize{1, 1, 2, 2};
-  // CHECK: store <4 x i8> <i8 12, i8 6, i8 8, i8 4>
+  static_assert(q[0] == 12 && q[1] == 6 && q[2] == 8 && q[3] == 4, "");
+
   constexpr auto r = FourCharsExtVec{19, 15, 12, 10} >>
                      FourCharsExtVec{1, 1, 2, 2};
-  // CHECK: store <4 x i8> <i8 9, i8 7, i8 3, i8 2>
+  static_assert(r[0] == 9 && r[1] == 7 && r[2] == 3 && r[3] == 2, "");
+
   constexpr auto s = FourCharsExtVec{6, 3, 5, 10} << 1;
-  // CHECK: store <4 x i8> <i8 12, i8 6, i8 10, i8 20>
+  static_assert(s[0] == 12 && s[1] == 6 && s[2] == 10 && s[3] == 20, "");
+
   constexpr auto t = FourCharsExtVec{19, 15, 10, 20} >> 1;
-  // CHECK: store <4 x i8> <i8 9, i8 7, i8 5, i8 10>
+  static_assert(t[0] == 9 && t[1] == 7 && t[2] == 5 && t[3] == 10, "");
+
   constexpr auto u = 12 << FourCharsExtVec{1, 2, 3, 3};
-  // CHECK: store <4 x i8> <i8 24, i8 48, i8 96, i8 96>
+  static_assert(u[0] == 24 && u[1] == 48 && u[2] == 96 && u[3] == 96, "");
+
   constexpr auto v = 12 >> FourCharsExtVec{1, 2, 2, 1};
-  // CHECK: store <4 x i8> <i8 6, i8 3, i8 3, i8 6>
+  static_assert(v[0] == 6 && v[1] == 3 && v[2] == 3 && v[3] == 6, "");
 
   constexpr auto w = FourCharsExtVec{1, 2, 3, 4} <
                      FourCharsExtVec{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 0>
+  static_assert(w[0] == -1 && w[1] == -1 && w[2] == 0 && w[3] == 0, "");
+
   constexpr auto x = FourCharsExtVec{1, 2, 3, 4} >
                      FourCharsExtVec{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(x[0] == 0 && x[1] == 0 && x[2] == -1 && x[3] == -1, "");
+
   constexpr auto y = FourCharsExtVec{1, 2, 3, 4} <=
                      FourCharsExtVec{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 -1, i8 0>
+  static_assert(y[0] == -1 && y[1] == -1 && y[2] == -1 && y[3] == 0, "");
+
   constexpr auto z = FourCharsExtVec{1, 2, 3, 4} >=
                      FourCharsExtVec{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(z[0] == 0 && z[1] == 0 && z[2] == -1 && z[3] == -1, "");
+
   constexpr auto A = FourCharsExtVec{1, 2, 3, 4} ==
                      FourCharsExtVec{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 0>
+  static_assert(A[0] == 0 && A[1] == 0 && A[2] == -1 && A[3] == 0, "");
+
   constexpr auto B = FourCharsExtVec{1, 2, 3, 4} !=
                      FourCharsExtVec{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 -1>
+  static_assert(B[0] == -1 && B[1] == -1 && B[2] == 0 && B[3] == -1, "");
 
   constexpr auto C = FourCharsExtVec{1, 2, 3, 4} < 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 0>
+  static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
+
   constexpr auto D = FourCharsExtVec{1, 2, 3, 4} > 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 0, i8 -1>
+  static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
+
   constexpr auto E = FourCharsExtVec{1, 2, 3, 4} <= 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 -1, i8 0>
+  static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
+
   constexpr auto F = FourCharsExtVec{1, 2, 3, 4} >= 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
+
   constexpr auto G = FourCharsExtVec{1, 2, 3, 4} == 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 0>
+  static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
+
   constexpr auto H = FourCharsExtVec{1, 2, 3, 4} != 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 -1>
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");
 
   constexpr auto I = FourCharsExtVec{1, 2, 3, 4} &
                      FourCharsExtVec{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 2, i8 2, i8 0>
+  static_assert(I[0] == 0 && I[1] == 2 && I[2] == 2 && I[3] == 0, "");
+
   constexpr auto J = FourCharsExtVec{1, 2, 3, 4} ^
                      FourCharsExtVec { 4, 3, 2, 1 };
-  // CHECK: store <4 x i8> <i8 5, i8 1, i8 1, i8 5>
+  static_assert(J[0] == 5 && J[1] == 1 && J[2] == 1 && J[3] == 5, "");
+
   constexpr auto K = FourCharsExtVec{1, 2, 3, 4} |
                      FourCharsExtVec{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 5, i8 3, i8 3, i8 5>
+  static_assert(K[0] == 5 && K[1] == 3 && K[2] == 3 && K[3] == 5, "");
+
   constexpr auto L = FourCharsExtVec{1, 2, 3, 4} & 3;
-  // CHECK: store <4 x i8> <i8 1, i8 2, i8 3, i8 0>
+  static_assert(L[0] == 1 && L[1] == 2 && L[2] == 3 && L[3] == 0, "");
+
   constexpr auto M = FourCharsExtVec{1, 2, 3, 4} ^ 3;
-  // CHECK: store <4 x i8> <i8 2, i8 1, i8 0, i8 7>
+  static_assert(M[0] == 2 && M[1] == 1 && M[2] == 0 && M[3] == 7, "");
+
   constexpr auto N = FourCharsExtVec{1, 2, 3, 4} | 3;
-  // CHECK: store <4 x i8> <i8 3, i8 3, i8 3, i8 7>
+  static_assert(N[0] == 3 && N[1] == 3 && N[2] == 3 && N[3] == 7, "");
 
   constexpr auto O = FourCharsExtVec{5, 0, 6, 0} &&
                      FourCharsExtVec{5, 5, 0, 0};
-  // CHECK: store <4 x i8> <i8 1, i8 0, i8 0, i8 0>
+  static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
+
   constexpr auto P = FourCharsExtVec{5, 0, 6, 0} ||
                      FourCharsExtVec{5, 5, 0, 0};
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+  static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
 
   constexpr auto Q = FourCharsExtVec{5, 0, 6, 0} && 3;
-  // CHECK: store <4 x i8> <i8 1, i8 0, i8 1, i8 0>
+  static_assert(Q[0] == 1 && Q[1] == 0 && Q[2] == 1 && Q[3] == 0, "");
+
   constexpr auto R = FourCharsExtVec{5, 0, 6, 0} || 3;
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 1>
+  static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  // CHECK: store <4 x i8> <i8 108, i8 18, i8 56, i8 72>
+  static_assert(T[0] == 108 && T[1] == 18 && T[2] == 56 && T[3] == 72, "");
 
   constexpr auto U = CmpDiv(a, b);
-  // CHECK: store <4 x i8> <i8 3, i8 18, i8 0, i8 0>
+  static_assert(U[0] == 3 && U[1] == 18 && U[2] == 0 && U[3] == 0, "");
 
   constexpr auto V = CmpRem(a, b);
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 7, i8 8>
+  static_assert(V[0] == 0 && V[1] == 0 && V[2] == 7 && V[3] == 8, "");
 
   constexpr auto X = CmpAdd(a, b);
-  // CHECK: store <4 x i8> <i8 24, i8 19, i8 15, i8 17>
+  static_assert(X[0] == 24 && X[1] == 19 && X[2] == 15 && X[3] == 17, "");
 
   constexpr auto Y = CmpSub(a, b);
-  // CHECK: store <4 x i8> <i8 12, i8 17, i8 -1, i8 -1>
+  static_assert(Y[0] == 12 && Y[1] == 17 && Y[2] == -1 && Y[3] == -1, "");
 
   constexpr auto InvH = -H;
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 1>
+  static_assert(InvH[0] == 1 && InvH[1] == 1 && InvH[2] == 0 && InvH[3] == 1, "");
 
   constexpr auto Z = CmpLSH(a, InvH);
-  // CHECK: store <4 x i8> <i8 36, i8 36, i8 7, i8 16>
+  static_assert(Z[0] == 36 && Z[1] == 36 && Z[2] == 7 && Z[3] == 16, "");
 
   constexpr auto aa = CmpRSH(a, InvH);
-  // CHECK: store <4 x i8> <i8 9, i8 9, i8 7, i8 4>
+  static_assert(aa[0] == 9 && aa[1] == 9 && aa[2] == 7 && aa[3] == 4, "");
 
   constexpr auto ab = CmpBinAnd(a, b);
-  // CHECK: store <4 x i8> <i8 2, i8 0, i8 0, i8 8>
+  static_assert(ab[0] == 2 && ab[1] == 0 && ab[2] == 0 && ab[3] == 8, "");
 
   constexpr auto ac = CmpBinXOr(a, b);
-  // CHECK: store <4 x i8> <i8 20, i8 19, i8 15, i8 1>
+  static_assert(ac[0] == 20 && ac[1] == 19 && ac[2] == 15 && ac[3] == 1, "");
 
   constexpr auto ad = CmpBinOr(a, b);
-  // CHECK: store <4 x i8> <i8 22, i8 19, i8 15, i8 9>
+  static_assert(ad[0] == 22 && ad[1] == 19 && ad[2] == 15 && ad[3] == 9, "");
 
   constexpr auto ae = ~FourCharsExtVec{1, 2, 10, 20};
-  // CHECK: store <4 x i8> <i8 -2, i8 -3, i8 -11, i8 -21>
+  static_assert(ae[0] == -2 && ae[1] == -3 && ae[2] == -11 && ae[3] == -21, "");
 
   constexpr auto af = !FourCharsExtVec{0, 1, 8, -1};
-  // CHECK: store <4 x i8> <i8 -1, i8 0, i8 0, i8 0>
+  static_assert(af[0] == -1 && af[1] == 0 && af[2] == 0 && af[3] == 0, "");
 }
 
 void FloatUsage() {
   constexpr auto a = FourFloatsVecSize{6, 3, 2, 1} +
                      FourFloatsVecSize{12, 15, 5, 7};
-  // CHECK: <4 x float> <float 1.800000e+01, float 1.800000e+01, float 7.000000e+00, float 8.000000e+00>
+  static_assert(a[0] == 1.800000e+01 && a[1] == 1.800000e+01 && a[2] == 7.000000e+00 && a[3] == 8.000000e+00, "");
+
   constexpr auto b = FourFloatsVecSize{19, 15, 13, 12} -
                      FourFloatsVecSize{13, 14, 5, 3};
-  // CHECK: store <4 x float> <float 6.000000e+00, float 1.000000e+00, float 8.000000e+00, float 9.000000e+00>
+  static_assert(b[0] == 6.000000e+00 && b[1] == 1.000000e+00 && b[2] == 8.000000e+00 && b[3] == 9.000000e+00, "");
+
   constexpr auto c = FourFloatsVecSize{8, 4, 2, 1} *
                      FourFloatsVecSize{3, 4, 5, 6};
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.600000e+01, float 1.000000e+01, float 6.000000e+00>
+  static_assert(c[0] == 2.400000e+01 && c[1] == 1.600000e+01 && c[2] == 1.000000e+01 && c[3] == 6.000000e+00, "");
+
   constexpr auto d = FourFloatsVecSize{12, 12, 10, 10} /
                      FourFloatsVecSize{6, 4, 5, 2};
-  // CHECK: store <4 x float> <float 2.000000e+00, float 3.000000e+00, float 2.000000e+00, float 5.000000e+00>
+  static_assert(d[0] == 2.000000e+00 && d[1] == 3.000000e+00 && d[2] == 2.000000e+00 && d[3] == 5.000000e+00, "");
 
   constexpr auto f = FourFloatsVecSize{6, 3, 2, 1} + 3;
-  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  static_assert(f[0] == 9.000000e+00 && f[1] == 6.000000e+00 && f[2] == 5.000000e+00 && f[3] == 4.000000e+00, "");
+
   constexpr auto g = FourFloatsVecSize{19, 15, 12, 10} - 3;
-  // CHECK: store <4 x float> <float 1.600000e+01, float 1.200000e+01, float 9.000000e+00, float 7.000000e+00>
+  static_assert(g[0] == 1.600000e+01 && g[1] == 1.200000e+01 && g[2] == 9.000000e+00 && g[3] == 7.000000e+00, "");
+
   constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  static_assert(h[0] == 2.400000e+01 && h[1] == 1.200000e+01 && h[2] == 6.000000e+00 && h[3] == 3.000000e+00, "");
+
   constexpr auto j = FourFloatsVecSize{12, 15, 18, 21} / 3;
-  // CHECK: store <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>
+  static_assert(j[0] == 4.000000e+00 && j[1] == 5.000000e+00 && j[2] == 6.000000e+00 && j[3] == 7.000000e+00, "");
 
   constexpr auto l = 3 + FourFloatsVecSize{6, 3, 2, 1};
-  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  static_assert(l[0] == 9.000000e+00 && l[1] == 6.000000e+00 && l[2] == 5.000000e+00 && l[3] == 4.000000e+00, "");
+
   constexpr auto m = 20 - FourFloatsVecSize{19, 15, 12, 10};
-  // CHECK: store <4 x float> <float 1.000000e+00, float 5.000000e+00, float 8.000000e+00, float 1.000000e+01>
+  static_assert(m[0] == 1.000000e+00 && m[1] == 5.000000e+00 && m[2] == 8.000000e+00 && m[3] == 1.000000e+01, "");
+
   constexpr auto n = 3 * FourFloatsVecSize{8, 4, 2, 1};
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  static_assert(n[0] == 2.400000e+01 && n[1] == 1.200000e+01 && n[2] == 6.000000e+00 && n[3] == 3.000000e+00, "");
+
   constexpr auto o = 100 / FourFloatsVecSize{12, 15, 18, 21};
-  // CHECK: store <4 x float> <float 0x4020AAAAA0000000, float 0x401AAAAAA0000000, float 0x401638E380000000, float 0x40130C30C0000000>
+  static_assert(CmpF(o[0], 100.0 / 12) && CmpF(o[1], 100.0 / 15) && CmpF(o[2], 100.0 / 18) && CmpF(o[3], 100.0 / 21), "");
 
   constexpr auto w = FourFloatsVecSize{1, 2, 3, 4} <
                      FourFloatsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 0>
+  static_assert(w[0] == -1 && w[1] == -1 && w[2] == 0 && w[3] == 0, "");
+
   constexpr auto x = FourFloatsVecSize{1, 2, 3, 4} >
                      FourFloatsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(x[0] == 0 && x[1] == 0 && x[2] == -1 && x[3] == -1, "");
+
   constexpr auto y = FourFloatsVecSize{1, 2, 3, 4} <=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 0>
+  static_assert(y[0] == -1 && y[1] == -1 && y[2] == -1 && y[3] == 0, "");
+
   constexpr auto z = FourFloatsVecSize{1, 2, 3, 4} >=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(z[0] == 0 && z[1] == 0 && z[2] == -1 && z[3] == -1, "");
+
   constexpr auto A = FourFloatsVecSize{1, 2, 3, 4} ==
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 0>
+  static_assert(A[0] == 0 && A[1] == 0 && A[2] == -1 && A[3] == 0, "");
+
   constexpr auto B = FourFloatsVecSize{1, 2, 3, 4} !=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 -1>
+  static_assert(B[0] == -1 && B[1] == -1 && B[2] == 0 && B[3] == -1, "");
 
   constexpr auto C = FourFloatsVecSize{1, 2, 3, 4} < 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 0>
+  static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
+
   constexpr auto D = FourFloatsVecSize{1, 2, 3, 4} > 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 0, i32 -1>
+  static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
+
   constexpr auto E = FourFloatsVecSize{1, 2, 3, 4} <= 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 0>
+  static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
+
   constexpr auto F = FourFloatsVecSize{1, 2, 3, 4} >= 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
+
   constexpr auto G = FourFloatsVecSize{1, 2, 3, 4} == 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 0>
+  static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
+
   constexpr auto H = FourFloatsVecSize{1, 2, 3, 4} != 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 -1>
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");
 
   constexpr auto O = FourFloatsVecSize{5, 0, 6, 0} &&
                      FourFloatsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i32> <i32 1, i32 0, i32 0, i32 0>
+  static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
+
   constexpr auto P = FourFloatsVecSize{5, 0, 6, 0} ||
                      FourFloatsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+  static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
 
   constexpr auto Q = FourFloatsVecSize{5, 0, 6, 0} && 3;
-  // CHECK: store <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  static_assert(Q[0] == 1 && Q[1] == 0 && Q[2] == 1 && Q[3] == 0, "");
+
   constexpr auto R = FourFloatsVecSize{5, 0, 6, 0} || 3;
-  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  // CHECK: store <4 x float> <float 1.080000e+02, float 1.800000e+01, float 5.600000e+01, float 7.200000e+01>
+  static_assert(T[0] == 1.080000e+02 && T[1] == 1.800000e+01 && T[2] == 5.600000e+01 && T[3] == 7.200000e+01, "");
 
   constexpr auto U = CmpDiv(a, b);
-  // CHECK: store <4 x float> <float 3.000000e+00, float 1.800000e+01, float 8.750000e-01, float 0x3FEC71C720000000>
+  static_assert(CmpF(U[0], a[0] / b[0]) && CmpF(U[1], a[1] / b[1]) && CmpF(U[2], a[2] / b[2]) && CmpF(U[3], a[3] / b[3]), "");
 
   constexpr auto X = CmpAdd(a, b);
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.900000e+01, float 1.500000e+01, float 1.700000e+01>
+  static_assert(X[0] == 2.400000e+01 && X[1] == 1.900000e+01 && X[2] == 1.500000e+01 && X[3] == 1.700000e+01, "");
 
   constexpr auto Y = CmpSub(a, b);
-  // CHECK: store <4 x float> <float 1.200000e+01, float 1.700000e+01, float -1.000000e+00, float -1.000000e+00>
+  static_assert(Y[0] == 1.200000e+01 && Y[1] == 1.700000e+01 && Y[2] == -1.000000e+00 && Y[3] == -1.000000e+00, "");
 
   constexpr auto Z = -Y;
-  // CHECK: store <4 x float> <float -1.200000e+01, float -1.700000e+01, float 1.000000e+00, float 1.000000e+00>
+  static_assert(Z[0] == -1.200000e+01 && Z[1] == -1.700000e+01 && Z[2] == 1.000000e+00 && Z[3] == 1.000000e+00, "");
 
   // Operator ~ is illegal on floats, so no test for that.
   constexpr auto af = !FourFloatsVecSize{0, 1, 8, -1};
-  // CHECK: store <4 x i32> <i32 -1, i32 0, i32 0, i32 0>
+  static_assert(af[0] == -1 && af[1] == 0 && af[2] == 0 && af[3] == 0, "");
 }
 
 void FloatVecUsage() {
   constexpr auto a = FourFloatsVecSize{6, 3, 2, 1} +
                      FourFloatsVecSize{12, 15, 5, 7};
-  // CHECK: <4 x float> <float 1.800000e+01, float 1.800000e+01, float 7.000000e+00, float 8.000000e+00>
+  static_assert(a[0] == 1.800000e+01 && a[1] == 1.800000e+01 && a[2] == 7.000000e+00 && a[3] == 8.000000e+00, "");
+
   constexpr auto b = FourFloatsVecSize{19, 15, 13, 12} -
                      FourFloatsVecSize{13, 14, 5, 3};
-  // CHECK: store <4 x float> <float 6.000000e+00, float 1.000000e+00, float 8.000000e+00, float 9.000000e+00>
+  static_assert(b[0] == 6.000000e+00 && b[1] == 1.000000e+00 && b[2] == 8.000000e+00 && b[3] == 9.000000e+00, "");
+
   constexpr auto c = FourFloatsVecSize{8, 4, 2, 1} *
                      FourFloatsVecSize{3, 4, 5, 6};
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.600000e+01, float 1.000000e+01, float 6.000000e+00>
+  static_assert(c[0] == 2.400000e+01 && c[1] == 1.600000e+01 && c[2] == 1.000000e+01 && c[3] == 6.000000e+00, "");
+
   constexpr auto d = FourFloatsVecSize{12, 12, 10, 10} /
                      FourFloatsVecSize{6, 4, 5, 2};
-  // CHECK: store <4 x float> <float 2.000000e+00, float 3.000000e+00, float 2.000000e+00, float 5.000000e+00>
+  static_assert(d[0] == 2.000000e+00 && d[1] == 3.000000e+00 && d[2] == 2.000000e+00 && d[3] == 5.000000e+00, "");
 
   constexpr auto f = FourFloatsVecSize{6, 3, 2, 1} + 3;
-  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  static_assert(f[0] == 9.000000e+00 && f[1] == 6.000000e+00 && f[2] == 5.000000e+00 && f[3] == 4.000000e+00, "");
+
   constexpr auto g = FourFloatsVecSize{19, 15, 12, 10} - 3;
-  // CHECK: store <4 x float> <float 1.600000e+01, float 1.200000e+01, float 9.000000e+00, float 7.000000e+00>
+  static_assert(g[0] == 1.600000e+01 && g[1] == 1.200000e+01 && g[2] == 9.000000e+00 && g[3] == 7.000000e+00, "");
+
   constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  static_assert(h[0] == 2.400000e+01 && h[1] == 1.200000e+01 && h[2] == 6.000000e+00 && h[3] == 3.000000e+00, "");
+
   constexpr auto j = FourFloatsVecSize{12, 15, 18, 21} / 3;
-  // CHECK: store <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>
+  static_assert(j[0] == 4.000000e+00 && j[1] == 5.000000e+00 && j[2] == 6.000000e+00 && j[3] == 7.000000e+00, "");
 
   constexpr auto l = 3 + FourFloatsVecSize{6, 3, 2, 1};
-  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  static_assert(l[0] == 9.000000e+00 && l[1] == 6.000000e+00 && l[2] == 5.000000e+00 && l[3] == 4.000000e+00, "");
+
   constexpr auto m = 20 - FourFloatsVecSize{19, 15, 12, 10};
-  // CHECK: store <4 x float> <float 1.000000e+00, float 5.000000e+00, float 8.000000e+00, float 1.000000e+01>
+  static_assert(m[0] == 1.000000e+00 && m[1] == 5.000000e+00 && m[2] == 8.000000e+00 && m[3] == 1.000000e+01, "");
+
   constexpr auto n = 3 * FourFloatsVecSize{8, 4, 2, 1};
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  static_assert(n[0] == 2.400000e+01 && n[1] == 1.200000e+01 && n[2] == 6.000000e+00 && n[3] == 3.000000e+00, "");
+
   constexpr auto o = 100 / FourFloatsVecSize{12, 15, 18, 21};
-  // CHECK: store <4 x float> <float 0x4020AAAAA0000000, float 0x401AAAAAA0000000, float 0x401638E380000000, float 0x40130C30C0000000>
+  static_assert(CmpF(o[0], 100.0 / 12) && CmpF(o[1], 100.0 / 15) && CmpF(o[2], 100.0 / 18) && CmpF(o[3], 100.0 / 21), "");
 
   constexpr auto w = FourFloatsVecSize{1, 2, 3, 4} <
                      FourFloatsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 0>
+  static_assert(w[0] == -1 && w[1] == -1 && w[2] == 0 && w[3] == 0, "");
+
   constexpr auto x = FourFloatsVecSize{1, 2, 3, 4} >
                      FourFloatsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(x[0] == 0 && x[1] == 0 && x[2] == -1 && x[2] == -1, "");
+
   constexpr auto y = FourFloatsVecSize{1, 2, 3, 4} <=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 0>
+  static_assert(y[0] == -1 && y[1] == -1 && y[2] == -1 && y[3] == 0, "");
+
   constexpr auto z = FourFloatsVecSize{1, 2, 3, 4} >=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(z[0] == 0 && z[1] == 0 && z[2] == -1 && z[3] == -1, "");
+
   constexpr auto A = FourFloatsVecSize{1, 2, 3, 4} ==
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 0>
+  static_assert(A[0] == 0 && A[1] == 0 && A[2] == -1 && A[3] == 0, "");
+
   constexpr auto B = FourFloatsVecSize{1, 2, 3, 4} !=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 -1>
+  static_assert(B[0] == -1 && B[1] == -1 && B[2] == 0 && B[3] == -1, "");
 
   constexpr auto C = FourFloatsVecSize{1, 2, 3, 4} < 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 0>
+  static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
+
   constexpr auto D = FourFloatsVecSize{1, 2, 3, 4} > 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 0, i32 -1>
+  static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
+
   constexpr auto E = FourFloatsVecSize{1, 2, 3, 4} <= 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 0>
+  static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
+
   constexpr auto F = FourFloatsVecSize{1, 2, 3, 4} >= 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
+
   constexpr auto G = FourFloatsVecSize{1, 2, 3, 4} == 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 0>
+  static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
+
   constexpr auto H = FourFloatsVecSize{1, 2, 3, 4} != 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 -1>
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");
 
   constexpr auto O = FourFloatsVecSize{5, 0, 6, 0} &&
                      FourFloatsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i32> <i32 1, i32 0, i32 0, i32 0>
+  static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
+
   constexpr auto P = FourFloatsVecSize{5, 0, 6, 0} ||
                      FourFloatsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+  static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
 
   constexpr auto Q = FourFloatsVecSize{5, 0, 6, 0} && 3;
-  // CHECK: store <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  static_assert(Q[0] == 1 && Q[1] == 0 && Q[2] == 1 && Q[3] == 0, "");
+
   constexpr auto R = FourFloatsVecSize{5, 0, 6, 0} || 3;
-  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  // CHECK: store <4 x float> <float 1.080000e+02, float 1.800000e+01, float 5.600000e+01, float 7.200000e+01>
+  static_assert(T[0] == 1.080000e+02 && T[1] == 1.800000e+01 && T[2] == 5.600000e+01 && T[3] == 7.200000e+01, "");
 
   constexpr auto U = CmpDiv(a, b);
-  // CHECK: store <4 x float> <float 3.000000e+00, float 1.800000e+01, float 8.750000e-01, float 0x3FEC71C720000000>
+  static_assert(CmpF(U[0], a[0] / b[0]) && CmpF(U[1], a[1] / b[1]) && CmpF(U[2], a[2] / b[2]) && CmpF(U[3], a[3] / b[3]), "");
 
   constexpr auto X = CmpAdd(a, b);
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.900000e+01, float 1.500000e+01, float 1.700000e+01>
+  static_assert(X[0] == 2.400000e+01 && X[1] == 1.900000e+01 && X[2] == 1.500000e+01 && X[3] == 1.700000e+01, "");
 
   constexpr auto Y = CmpSub(a, b);
-  // CHECK: store <4 x float> <float 1.200000e+01, float 1.700000e+01, float -1.000000e+00, float -1.000000e+00>
+  static_assert(Y[0] == 1.200000e+01 && Y[1] == 1.700000e+01 && Y[2] == -1.000000e+00 && Y[3] == -1.000000e+00, "");
 
   constexpr auto Z = -Y;
-  // CHECK: store <4 x float> <float -1.200000e+01, float -1.700000e+01, float 1.000000e+00, float 1.000000e+00>
+  static_assert(Z[0] == -1.200000e+01 && Z[1] == -1.700000e+01 && Z[2] == 1.000000e+00 && Z[3] == 1.000000e+00, "");
 
   // Operator ~ is illegal on floats, so no test for that.
   constexpr auto af = !FourFloatsVecSize{0, 1, 8, -1};
-  // CHECK: store <4 x i32> <i32 -1, i32 0, i32 0, i32 0>
+  static_assert(af[0] == -1 && af[1] == 0 && af[2] == 0 && af[3] == 0, "");
 }
 
 void I128Usage() {
   constexpr auto a = FourI128VecSize{1, 2, 3, 4};
-  // CHECK: store <4 x i128> <i128 1, i128 2, i128 3, i128 4>
+  static_assert(a[0] == 1 && a[1] == 2 && a[2] == 3 && a[3] == 4, "");
+
   constexpr auto b = a < 3;
-  // CHECK: store <4 x i128> <i128 -1, i128 -1, i128 0, i128 0>
+  static_assert(b[0] == -1 && b[1] == -1 && b[2] == 0 && b[3] == 0, "");
 
   // Operator ~ is illegal on floats, so no test for that.
   constexpr auto c = ~FourI128VecSize{1, 2, 10, 20};
-  // CHECK: store <4 x i128> <i128 -2, i128 -3, i128 -11, i128 -21>
+  static_assert(c[0] == -2 && c[1] == -3 && c[2] == -11 && c[3] == -21, "");
 
   constexpr auto d = !FourI128VecSize{0, 1, 8, -1};
-  // CHECK: store <4 x i128> <i128 -1, i128 0, i128 0, i128 0>
+  static_assert(d[0] == -1 && d[1] == 0 && d[2] == 0 && d[3] == 0, "");
 }
 
 void I128VecUsage() {
   constexpr auto a = FourI128ExtVec{1, 2, 3, 4};
-  // CHECK: store <4 x i128> <i128 1, i128 2, i128 3, i128 4>
+  static_assert(a[0] == 1 && a[1] == 2 && a[2] == 3 && a[3] == 4, "");
+
   constexpr auto b = a < 3;
-  // CHECK: store <4 x i128> <i128 -1, i128 -1, i128 0, i128 0>
+  static_assert(b[0] == -1 && b[1] == -1 && b[2] == 0 && b[3] == 0, "");
 
   // Operator ~ is illegal on floats, so no test for that.
   constexpr auto c = ~FourI128ExtVec{1, 2, 10, 20};
-  // CHECK: store <4 x i128>  <i128 -2, i128 -3, i128 -11, i128 -21>
+  static_assert(c[0] == -2 && c[1] == -3 && c[2] == -11 && c[3] == -21, "");
 
   constexpr auto d = !FourI128ExtVec{0, 1, 8, -1};
-  // CHECK: store <4 x i128>  <i128 -1, i128 0, i128 0, i128 0>
+  static_assert(d[0] == -1 && d[1] == 0 && d[2] == 0 && d[3] == 0, "");
 }
 
 using FourBoolsExtVec __attribute__((ext_vector_type(4))) = bool;
 void BoolVecUsage() {
   constexpr auto a = FourBoolsExtVec{true, false, true, false} <
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 false, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %a, align 1
+  static_assert(a[0] == false && a[1] == false && a[2] == false && a[3] == true, "");
+
   constexpr auto b = FourBoolsExtVec{true, false, true, false} <=
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 true, i1 true, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %b, align 1
+  static_assert(b[0] == false && b[1] == true && b[2] == true && b[3] == true, "");
+
   constexpr auto c = FourBoolsExtVec{true, false, true, false} ==
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 true, i1 true, i1 false, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %c, align 1
+  static_assert(c[0] == false && c[1] == true && c[2] == true && c[3] == false, "");
+
   constexpr auto d = FourBoolsExtVec{true, false, true, false} !=
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 false, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %d, align 1
+  static_assert(d[0] == true && d[1] == false && d[2] == false && d[3] == true, "");
+
   constexpr auto e = FourBoolsExtVec{true, false, true, false} >=
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 true, i1 true, i1 false, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %e, align 1
+  static_assert(e[0] == true && e[1] == true && e[2] == true && e[3] == false, "");
+
   constexpr auto f = FourBoolsExtVec{true, false, true, false} >
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %f, align 1
+  static_assert(f[0] == true && f[1] == false && f[2] == false && f[3] == false, "");
+
   constexpr auto g = FourBoolsExtVec{true, false, true, false} &
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 false, i1 true, i1 false, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %g, align 1
+  static_assert(g[0] == false && g[1] == false && g[2] == true && g[3] == false, "");
+
   constexpr auto h = FourBoolsExtVec{true, false, true, false} |
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 false, i1 true, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %h, align 1
+  static_assert(h[0] == true && h[1] == false && h[2] == true && h[3] == true, "");
+
   constexpr auto i = FourBoolsExtVec{true, false, true, false} ^
                      FourBoolsExtVec { false, false, true, true };
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 false, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %i, align 1
+  static_assert(i[0] == true && i[1] == false && i[2] == false && i[3] == true, "");
+
   constexpr auto j = !FourBoolsExtVec{true, false, true, false};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 true, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %j, align 1
+  static_assert(j[0] == false && j[1] == true && j[2] == false && j[3] == true, "");
+
   constexpr auto k = ~FourBoolsExtVec{true, false, true, false};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 true, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %k, align 1
+  static_assert(k[0] == false && k[1] == true && k[2] == false && k[3] == true, "");
 }
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index 45fee6514c12bc6..bfcc5991cee5aca 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -729,10 +729,10 @@ struct S2 {
 };
 
 S2& S2::operator=(this int&& self, const S2&) = default;
-// expected-error@-1 {{the type of the explicit object parameter of an explicitly-defaulted copy assignment operator should match the type of the class 'S2'}}
+// expected-error@-1 {{the type of the explicit object parameter of an explicitly-defaulted copy assignment operator should be reference to 'S2'}}
 
 S2& S2::operator=(this int&& self, S2&&) = default;
-// expected-error@-1 {{the type of the explicit object parameter of an explicitly-defaulted move assignment operator should match the type of the class 'S2'}}
+// expected-error@-1 {{the type of the explicit object parameter of an explicitly-defaulted move assignment operator should be reference to 'S2'}}
 
 struct Move {
     Move& operator=(this int&, Move&&) = default;
@@ -972,3 +972,61 @@ struct R {
 	f(r_value_ref); // expected-error {{no matching member function for call to 'f'}}
   }
 };
+
+namespace GH100329 {
+struct A {
+    bool operator == (this const int&, const A&);
+};
+bool A::operator == (this const int&, const A&) = default;
+// expected-error@-1 {{invalid parameter type for defaulted equality comparison operator; found 'const int &', expected 'const GH100329::A &'}}
+} // namespace GH100329
+
+namespace defaulted_assign {
+struct A {
+  A& operator=(this A, const A&) = default;
+  // expected-warning@-1 {{explicitly defaulted copy assignment operator is implicitly deleted}}
+  // expected-note@-2 {{function is implicitly deleted because its declared type does not match the type of an implicit copy assignment operator}}
+  A& operator=(this int, const A&) = default;
+  // expected-warning@-1 {{explicitly defaulted copy assignment operator is implicitly deleted}}
+  // expected-note@-2 {{function is implicitly deleted because its declared type does not match the type of an implicit copy assignment operator}}
+};
+} // namespace defaulted_assign
+
+namespace defaulted_compare {
+struct A {
+  bool operator==(this A&, const A&) = default;
+  // expected-error@-1 {{defaulted member equality comparison operator must be const-qualified}}
+  bool operator==(this const A, const A&) = default;
+  // expected-error@-1 {{invalid parameter type for defaulted equality comparison operator; found 'const A', expected 'const defaulted_compare::A &'}}
+  bool operator==(this A, A) = default;
+};
+struct B {
+  int a;
+  bool operator==(this B, B) = default;
+};
+static_assert(B{0} == B{0});
+static_assert(B{0} != B{1});
+template<B b>
+struct X;
+static_assert(__is_same(X<B{0}>, X<B{0}>));
+static_assert(!__is_same(X<B{0}>, X<B{1}>));
+} // namespace defaulted_compare
+
+namespace GH102025 {
+struct Foo {
+  template <class T>
+  constexpr auto operator[](this T &&self, auto... i) // expected-note {{candidate template ignored: substitution failure [with T = Foo &, i:auto = <>]: member '_evaluate' used before its declaration}}
+      -> decltype(_evaluate(self, i...)) {
+    return self._evaluate(i...);
+  }
+
+private:
+  template <class T>
+  constexpr auto _evaluate(this T &&self, auto... i) -> decltype((i + ...));
+};
+
+int main() {
+  Foo foo;
+  return foo[]; // expected-error {{no viable overloaded operator[] for type 'Foo'}}
+}
+}
diff --git a/clang/test/SemaCXX/cxx2c-variadic-friends.cpp b/clang/test/SemaCXX/cxx2c-variadic-friends.cpp
new file mode 100644
index 000000000000000..a4d7c8078338d2a
--- /dev/null
+++ b/clang/test/SemaCXX/cxx2c-variadic-friends.cpp
@@ -0,0 +1,156 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2c %s
+
+struct A;
+struct B;
+struct C;
+
+struct S {};
+template <typename> struct TS {};
+
+template <typename ...Pack>
+class X {
+  friend Pack...;
+  static void f() { } // expected-note {{declared private here}}
+};
+
+class Y {
+  friend A, B, C;
+  static void g() { } // expected-note {{declared private here}}
+};
+
+struct A {
+  A() {
+    X<A>::f();
+    Y::g();
+  };
+};
+
+struct B {
+  B() {
+    X<B, C>::f();
+    Y::g();
+  };
+};
+
+struct C {
+  C() {
+    X<A, B, C>::f();
+    Y::g();
+  };
+};
+
+struct D {
+  D() {
+    X<A, B, C>::f(); // expected-error {{'f' is a private member of 'X<A, B, C>'}}
+    Y::g(); // expected-error {{'g' is a private member of 'Y'}}
+  };
+};
+
+void f1() {
+  A a;
+  B b;
+  C c;
+  D d;
+}
+
+template <typename ...Pack>
+struct Z {
+  template <template <typename> class Template>
+  struct Inner {
+    friend Template<Pack>...;
+  };
+};
+
+void f2() {
+  Z<int, long, char> z;
+  Z<int, long, char>::Inner<TS> inner;
+}
+
+namespace p2893r3_examples {
+template<class... Ts>
+class Passkey {
+  friend Ts...;
+  Passkey() {} // expected-note {{declared private here}}
+};
+
+class Foo;
+class Bar;
+class Baz;
+
+class C {
+public:
+  void f(Passkey<Foo, Bar, Baz>);
+};
+
+class Foo {
+  Foo() { C c; c.f({}); }
+};
+
+class Bar {
+  Bar() { C c; c.f({}); }
+};
+
+class Baz {
+  Baz() { C c; c.f({}); }
+};
+
+class Quux {
+  Quux() { C c; c.f({}); } // expected-error {{calling a private constructor of class 'p2893r3_examples::Passkey<p2893r3_examples::Foo, p2893r3_examples::Bar, p2893r3_examples::Baz>'}}
+};
+
+template<class Derived, class MsgT>
+struct Receiver {
+  void receive(MsgT) {
+    static_cast<Derived*>(this)->private_ += 1;
+  }
+};
+
+template<class... MsgTs>
+struct Dispatcher : Receiver<Dispatcher<MsgTs...>, MsgTs>... {
+  using Receiver<Dispatcher, MsgTs>::receive...;
+  friend Receiver<Dispatcher, MsgTs>...;
+
+private:
+  int private_;
+};
+
+void f() {
+  Dispatcher<int, float> d;
+  d.receive(0);
+  d.receive(0.0f);
+}
+} // namespace p2893r3_examples
+
+namespace p2893r3_note {
+template <class... Ts> class R {
+  friend Ts...;
+};
+
+template <class... Ts, class... Us>
+class R<R<Ts...>, R<Us...>> {
+  friend Ts::Nested..., Us...;
+};
+
+struct E { struct Nested; };
+R<R<E>, R<C, int>> rr;
+} // namespace p2893r3_note
+
+namespace template_template {
+template <typename U, template <typename> typename... Friend>
+class S {
+  friend class Friend<U>...;
+  static constexpr int a = 42;
+};
+
+template <typename U>
+struct T {
+  static_assert(S<U, T>::a == 42);
+  static_assert(S<U, T>::a == 43); // expected-error {{static assertion failed due to requirement 'S<int, template_template::T>::a == 43'}} \
+                                   // expected-note {{expression evaluates to '42 == 43'}}
+};
+
+void f() {
+  T<int> t; // expected-note {{in instantiation of}}
+}
+}
+
diff --git a/clang/test/SemaCXX/gh102293.cpp b/clang/test/SemaCXX/gh102293.cpp
index 30629fc03bf6a9a..d4218cc13dcecdc 100644
--- a/clang/test/SemaCXX/gh102293.cpp
+++ b/clang/test/SemaCXX/gh102293.cpp
@@ -1,5 +1,4 @@
 // RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify %s
-// expected-no-diagnostics
 
 template <typename T> static void destroy() {
     T t;
@@ -20,3 +19,29 @@ struct S : HasVT {
   HasD<> v;
 };
 
+// Ensure we don't get infinite recursion from the check, however. See GH104802
+namespace GH104802 {
+class foo {       // expected-note {{definition of 'GH104802::foo' is not complete until the closing '}'}}
+  foo a;          // expected-error {{field has incomplete type 'foo'}}
+
+  virtual int c();
+};
+
+class bar {       // expected-note {{definition of 'GH104802::bar' is not complete until the closing '}'}}
+  const bar a;    // expected-error {{field has incomplete type 'const bar'}}
+
+  virtual int c();
+};
+
+class baz {       // expected-note {{definition of 'GH104802::baz' is not complete until the closing '}'}}
+  typedef class baz blech;
+  blech a;        // expected-error {{field has incomplete type 'blech' (aka 'GH104802::baz')}}
+
+  virtual int c();
+};
+
+class quux : quux { // expected-error {{base class has incomplete type}} \
+                     expected-note {{definition of 'GH104802::quux' is not complete until the closing '}'}}
+  virtual int c();
+};
+}
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
index a1ddc384e0d9b8c..e97511593bbd815 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
@@ -79,6 +79,8 @@ namespace construct_wt_ptr_size {
     unsigned Y = 10;
     std::span<int> S = std::span{&X, 1}; // no-warning
     int Arr[10];
+    typedef int TenInts_t[10];
+    TenInts_t Arr2;
 
     S = std::span{&X, 2};                // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
     S = std::span{new int[10], 10};      // no-warning
@@ -90,6 +92,7 @@ namespace construct_wt_ptr_size {
     S = std::span{new int[10], 9};       // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}  // not smart enough to tell its safe
     S = std::span{new int[10], Y};       // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}  // not smart enough to tell its safe
     S = std::span{Arr, 10};              // no-warning
+    S = std::span{Arr2, 10};             // no-warning
     S = std::span{Arr, Y};               // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}  // not smart enough to tell its safe
     S = std::span{p, 0};                 // no-warning
   }
diff --git a/clang/test/SemaHLSL/Types/typedefs.hlsl b/clang/test/SemaHLSL/Types/typedefs.hlsl
new file mode 100644
index 000000000000000..fd72b1ae8a47ffe
--- /dev/null
+++ b/clang/test/SemaHLSL/Types/typedefs.hlsl
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.4-library -finclude-default-header -verify -fnative-half-type %s
+// RUN: %clang_cc1 -triple spirv-linux-vulkan-library -finclude-default-header -verify -fnative-half-type %s
+
+// expected-no-diagnostics
+#define SizeCheck(Ty, SizeInBits)                                              \
+  _Static_assert(sizeof(Ty) == SizeInBits / 8, #Ty " is " #SizeInBits "-bit"); \
+  _Static_assert(sizeof(Ty##1) == (SizeInBits * 1) / 8, #Ty "1 is 1x" #SizeInBits "-bit"); \
+  _Static_assert(__builtin_vectorelements(Ty##1) == 1, #Ty "1 is has 1 " #SizeInBits "-bit element"); \
+  _Static_assert(sizeof(Ty##2) == (SizeInBits * 2) / 8, #Ty "2 is 2x" #SizeInBits "-bit"); \
+  _Static_assert(__builtin_vectorelements(Ty##2) == 2, #Ty "2 is has 2 " #SizeInBits "-bit element"); \
+  _Static_assert(__builtin_vectorelements(Ty##3) == 3, #Ty "3 is has 3 " #SizeInBits "-bit element"); \
+  _Static_assert(sizeof(Ty##4) == (SizeInBits * 4) / 8, #Ty "4 is 4x" #SizeInBits "-bit"); \
+  _Static_assert(__builtin_vectorelements(Ty##4) == 4, #Ty "4 is has 4 " #SizeInBits "-bit element");
+
+// FIXME: https://github.com/llvm/llvm-project/issues/104503 - 3 element vectors
+// should be the size of 3 elements not padded to 4.
+// _Static_assert(sizeof(Ty##3) == (SizeInBits * 3) / 8, #Ty "3 is 3x" #SizeInBits "-bit");
+
+SizeCheck(int16_t, 16);
+SizeCheck(uint16_t, 16);
+SizeCheck(half, 16);
+SizeCheck(float16_t, 16);
+
+SizeCheck(int, 32);
+SizeCheck(uint, 32);
+SizeCheck(int32_t, 32);
+SizeCheck(uint32_t, 32);
+SizeCheck(float, 32);
+SizeCheck(float32_t, 32);
+
+SizeCheck(int64_t, 64);
+SizeCheck(uint64_t, 64);
+SizeCheck(double, 64);
+SizeCheck(float64_t, 64);
diff --git a/clang/test/SemaOpenMP/gh104810.cpp b/clang/test/SemaOpenMP/gh104810.cpp
new file mode 100644
index 000000000000000..926408931977604
--- /dev/null
+++ b/clang/test/SemaOpenMP/gh104810.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -fopenmp -fsyntax-only %s
+
+// expected-no-diagnostics
+struct S {
+  int i;
+};
+
+auto [a] = S{1};
+
+void foo() {
+    a;
+}
diff --git a/clang/test/SemaTemplate/explicit-instantiation-cxx20.cpp b/clang/test/SemaTemplate/explicit-instantiation-cxx20.cpp
new file mode 100644
index 000000000000000..33500baf1e37b10
--- /dev/null
+++ b/clang/test/SemaTemplate/explicit-instantiation-cxx20.cpp
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++20  %s
+
+template<class T>
+concept C = true;
+
+template<class T>
+concept D = C<T> && true;
+
+template<typename T>
+struct a {
+    void no_candidate() requires(false) {}
+    // expected-note@-1 {{candidate function not viable: constraints not satisfied}} \
+    // expected-note@-1 {{because 'false' evaluated to false}}
+    void no_candidate() requires(false && false) {}
+    // expected-note@-1 {{candidate function not viable: constraints not satisfied}} \
+    // expected-note@-1 {{because 'false' evaluated to false}}
+
+    void subsumes();
+    void subsumes() requires C<T>;
+    void subsumes() requires D<T> {};
+
+    void ok() requires false;
+    void ok() requires true {};
+
+    void ok2() requires false;
+    void ok2(){};
+
+    void ambiguous() requires true;
+    // expected-note@-1 {{candidate function}}
+    void ambiguous() requires C<T>;
+    // expected-note@-1 {{candidate function}}
+};
+template void a<int>::no_candidate();
+// expected-error@-1 {{no viable candidate for explicit instantiation of 'no_candidate'}}
+
+template void a<int>::ambiguous();
+// expected-error@-1 {{partial ordering for explicit instantiation of 'ambiguous' is ambiguous}}
+
+template void a<int>::ok();
+template void a<int>::ok2();
diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 78019d39742e64d..070cf8b44e8eb63 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -96,10 +96,6 @@ struct AssemblerInvocation {
   LLVM_PREFERRED_TYPE(bool)
   unsigned GenDwarfForAssembly : 1;
   LLVM_PREFERRED_TYPE(bool)
-  unsigned RelaxELFRelocations : 1;
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned SSE2AVX : 1;
-  LLVM_PREFERRED_TYPE(bool)
   unsigned Dwarf64 : 1;
   unsigned DwarfVersion;
   std::string DwarfDebugFlags;
@@ -169,6 +165,11 @@ struct AssemblerInvocation {
   LLVM_PREFERRED_TYPE(bool)
   unsigned Crel : 1;
 
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned X86RelaxRelocations : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned X86Sse2Avx : 1;
+
   /// The name of the relocation model to use.
   std::string RelocationModel;
 
@@ -199,7 +200,6 @@ struct AssemblerInvocation {
     ShowInst = 0;
     ShowEncoding = 0;
     RelaxAll = 0;
-    SSE2AVX = 0;
     NoExecStack = 0;
     FatalWarnings = 0;
     NoWarn = 0;
@@ -211,6 +211,8 @@ struct AssemblerInvocation {
     EmitDwarfUnwind = EmitDwarfUnwindType::Default;
     EmitCompactUnwindNonCanonical = false;
     Crel = false;
+    X86RelaxRelocations = 0;
+    X86Sse2Avx = 0;
   }
 
   static bool CreateFromArgs(AssemblerInvocation &Res,
@@ -290,8 +292,6 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
             .Default(llvm::DebugCompressionType::None);
   }
 
-  Opts.RelaxELFRelocations = !Args.hasArg(OPT_mrelax_relocations_no);
-  Opts.SSE2AVX = Args.hasArg(OPT_msse2avx);
   if (auto *DwarfFormatArg = Args.getLastArg(OPT_gdwarf64, OPT_gdwarf32))
     Opts.Dwarf64 = DwarfFormatArg->getOption().matches(OPT_gdwarf64);
   Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
@@ -382,6 +382,8 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
   Opts.EmitCompactUnwindNonCanonical =
       Args.hasArg(OPT_femit_compact_unwind_non_canonical);
   Opts.Crel = Args.hasArg(OPT_crel);
+  Opts.X86RelaxRelocations = !Args.hasArg(OPT_mrelax_relocations_no);
+  Opts.X86Sse2Avx = Args.hasArg(OPT_msse2avx);
 
   Opts.AsSecureLogFile = Args.getLastArgValue(OPT_as_secure_log_file);
 
@@ -440,8 +442,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
   MCOptions.EmitCompactUnwindNonCanonical = Opts.EmitCompactUnwindNonCanonical;
   MCOptions.MCSaveTempLabels = Opts.SaveTemporaryLabels;
   MCOptions.Crel = Opts.Crel;
-  MCOptions.X86RelaxRelocations = Opts.RelaxELFRelocations;
-  MCOptions.X86Sse2Avx = Opts.SSE2AVX;
+  MCOptions.X86RelaxRelocations = Opts.X86RelaxRelocations;
+  MCOptions.X86Sse2Avx = Opts.X86Sse2Avx;
   MCOptions.CompressDebugSections = Opts.CompressDebugSections;
   MCOptions.AsSecureLogFile = Opts.AsSecureLogFile;
 
diff --git a/clang/tools/libclang/CXSourceLocation.cpp b/clang/tools/libclang/CXSourceLocation.cpp
index 53cb71f7276f299..cd41e9f0d64031b 100644
--- a/clang/tools/libclang/CXSourceLocation.cpp
+++ b/clang/tools/libclang/CXSourceLocation.cpp
@@ -50,6 +50,18 @@ unsigned clang_equalLocations(CXSourceLocation loc1, CXSourceLocation loc2) {
           loc1.int_data == loc2.int_data);
 }
 
+unsigned clang_isBeforeInTranslationUnit(CXSourceLocation loc1,
+                                         CXSourceLocation loc2) {
+  const SourceLocation Loc1 = SourceLocation::getFromRawEncoding(loc1.int_data);
+  const SourceLocation Loc2 = SourceLocation::getFromRawEncoding(loc2.int_data);
+
+  const SourceManager &SM =
+      *static_cast<const SourceManager *>(loc1.ptr_data[0]);
+  // Use the appropriate SourceManager method here rather than operator< because
+  // ordering is meaningful only if LHS and RHS have the same FileID.
+  return SM.isBeforeInTranslationUnit(Loc1, Loc2);
+}
+
 CXSourceRange clang_getNullRange() {
   CXSourceRange Result = { { nullptr, nullptr }, 0, 0 };
   return Result;
diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map
index 371fe512ce71c1f..25d8ba57d325149 100644
--- a/clang/tools/libclang/libclang.map
+++ b/clang/tools/libclang/libclang.map
@@ -434,6 +434,11 @@ LLVM_19 {
     clang_Cursor_getBinaryOpcodeStr;
 };
 
+LLVM_20 {
+  global:
+    clang_isBeforeInTranslationUnit;
+};
+
 # Example of how to add a new symbol version entry.  If you do add a new symbol
 # version, please update the example to depend on the version you added.
 # LLVM_X {
diff --git a/clang/tools/scan-build/man/scan-build.1 b/clang/tools/scan-build/man/scan-build.1
index e2b37f6062dbc0b..b29e9ca91477f65 100644
--- a/clang/tools/scan-build/man/scan-build.1
+++ b/clang/tools/scan-build/man/scan-build.1
@@ -2,9 +2,9 @@
 .\" See https://llvm.org/LICENSE.txt for license information.
 .\" SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 .\" $Id$
-.Dd Feb 10, 2024
+.Dd Aug 18, 2024
 .Dt SCAN-BUILD 1
-.Os "clang" "19"
+.Os "clang" "20"
 .Sh NAME
 .Nm scan-build
 .Nd Clang static analyzer
diff --git a/clang/unittests/AST/Interp/CMakeLists.txt b/clang/unittests/AST/ByteCode/CMakeLists.txt
similarity index 100%
rename from clang/unittests/AST/Interp/CMakeLists.txt
rename to clang/unittests/AST/ByteCode/CMakeLists.txt
diff --git a/clang/unittests/AST/Interp/Descriptor.cpp b/clang/unittests/AST/ByteCode/Descriptor.cpp
similarity index 95%
rename from clang/unittests/AST/Interp/Descriptor.cpp
rename to clang/unittests/AST/ByteCode/Descriptor.cpp
index abcf1bb0714cc8c..9e09c7f15ac9a28 100644
--- a/clang/unittests/AST/Interp/Descriptor.cpp
+++ b/clang/unittests/AST/ByteCode/Descriptor.cpp
@@ -1,6 +1,6 @@
-#include "../../../lib/AST/Interp/Descriptor.h"
-#include "../../../lib/AST/Interp/Context.h"
-#include "../../../lib/AST/Interp/Program.h"
+#include "../../../lib/AST/ByteCode/Descriptor.h"
+#include "../../../lib/AST/ByteCode/Context.h"
+#include "../../../lib/AST/ByteCode/Program.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
@@ -15,17 +15,17 @@ using namespace clang::ast_matchers;
 /// Inspect generated Descriptors as well as the pointers we create.
 ///
 TEST(Descriptor, Primitives) {
-  constexpr char Code[] =
-      "struct A { bool a; bool b; };\n"
-      "struct S {\n"
-      "  float f;\n"
-      "  char s[4];\n"
-      "  A a[3];\n"
-      "  short l[3][3];\n"
-      "  int EmptyA[0];\n"
-      "};\n"
-      "constexpr S d = {0.0, \"foo\", {{true, false}, {false, true}, {false, false}},\n"
-      "  {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, {}};\n";
+  constexpr char Code[] = "struct A { bool a; bool b; };\n"
+                          "struct S {\n"
+                          "  float f;\n"
+                          "  char s[4];\n"
+                          "  A a[3];\n"
+                          "  short l[3][3];\n"
+                          "  int EmptyA[0];\n"
+                          "};\n"
+                          "constexpr S d = {0.0, \"foo\", {{true, false}, "
+                          "{false, true}, {false, false}},\n"
+                          "  {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, {}};\n";
 
   auto AST = tooling::buildASTFromCodeWithArgs(
       Code, {"-fexperimental-new-constant-interpreter"});
diff --git a/clang/unittests/AST/Interp/toAPValue.cpp b/clang/unittests/AST/ByteCode/toAPValue.cpp
similarity index 98%
rename from clang/unittests/AST/Interp/toAPValue.cpp
rename to clang/unittests/AST/ByteCode/toAPValue.cpp
index 5ec607a8243492d..3f141878fb9594d 100644
--- a/clang/unittests/AST/Interp/toAPValue.cpp
+++ b/clang/unittests/AST/ByteCode/toAPValue.cpp
@@ -1,6 +1,6 @@
-#include "../../../lib/AST/Interp/Context.h"
-#include "../../../lib/AST/Interp/Descriptor.h"
-#include "../../../lib/AST/Interp/Program.h"
+#include "../../../lib/AST/ByteCode/Context.h"
+#include "../../../lib/AST/ByteCode/Descriptor.h"
+#include "../../../lib/AST/ByteCode/Program.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
diff --git a/clang/unittests/AST/CMakeLists.txt b/clang/unittests/AST/CMakeLists.txt
index 51245599736cfbb..40d2e1ff77a6015 100644
--- a/clang/unittests/AST/CMakeLists.txt
+++ b/clang/unittests/AST/CMakeLists.txt
@@ -5,7 +5,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 
-add_subdirectory(Interp)
+add_subdirectory(ByteCode)
 
 add_clang_unittest(ASTTests
   ASTContextParentMapTest.cpp
diff --git a/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp b/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
index cd1c076ab09e6b4..834aa7f4c2ac2e1 100644
--- a/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
@@ -85,4 +85,26 @@ TEST(ASTOpsTest, ReferencedDeclsOnUnionInitList) {
               UnorderedElementsAre(IDecl));
 }
 
+TEST(ASTOpsTest, ReferencedDeclsLocalsNotParamsOrStatics) {
+  std::string Code = R"cc(
+    void func(int Param) {
+      static int Static = 0;
+      int Local = Param;
+      Local = Static;
+    }
+  )cc";
+  std::unique_ptr<ASTUnit> Unit =
+      tooling::buildASTFromCodeWithArgs(Code, {"-fsyntax-only", "-std=c++17"});
+  auto &ASTCtx = Unit->getASTContext();
+
+  ASSERT_EQ(ASTCtx.getDiagnostics().getClient()->getNumErrors(), 0U);
+
+  auto *Func = cast<FunctionDecl>(findValueDecl(ASTCtx, "func"));
+  ASSERT_NE(Func, nullptr);
+  auto *LocalDecl = cast<VarDecl>(findValueDecl(ASTCtx, "Local"));
+
+  EXPECT_THAT(getReferencedDecls(*Func).Locals,
+              UnorderedElementsAre(LocalDecl));
+}
+
 } // namespace
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index bad1b1d662d1330..794ccab37045346 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -25857,7 +25857,90 @@ TEST_F(FormatTest, RequiresClausesPositions) {
                "}",
                Style);
 
+  Style.RequiresClausePosition = FormatStyle::RCPS_OwnLineWithBrace;
+  Style.IndentRequiresClause = true;
+
+  verifyFormat("template <typename T>\n"
+               "  requires(Foo<T> && std::trait<T>)\n"
+               "struct Bar;",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "  requires(Foo<T> && std::trait<T>)\n"
+               "class Bar {\n"
+               "public:\n"
+               "  Bar(T t);\n"
+               "  bool baz();\n"
+               "};",
+               Style);
+
+  verifyFormat(
+      "template <typename T>\n"
+      "  requires requires(T &&t) {\n"
+      "             typename T::I;\n"
+      "             requires(F<typename T::I> && std::trait<typename T::I>);\n"
+      "           }\n"
+      "Bar(T) -> Bar<typename T::I>;",
+      Style);
+
+  verifyFormat("template <typename T>\n"
+               "  requires(Foo<T> && std::trait<T>)\n"
+               "constexpr T MyGlobal;",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "  requires Foo<T> && requires(T t) {\n"
+               "                       { t.baz() } -> std::same_as<bool>;\n"
+               "                       requires std::same_as<T::Factor, int>;\n"
+               "                     }\n"
+               "inline int bar(T t) {\n"
+               "  return t.baz() ? T::Factor : 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "inline int bar(T t)\n"
+               "  requires Foo<T> && requires(T t) {\n"
+               "                       { t.baz() } -> std::same_as<bool>;\n"
+               "                       requires std::same_as<T::Factor, int>;\n"
+               "                     } {\n"
+               "  return t.baz() ? T::Factor : 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "  requires F<T>\n"
+               "int bar(T t) {\n"
+               "  return 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "int bar(T t)\n"
+               "  requires F<T> {\n"
+               "  return 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "int S::bar(T t) &&\n"
+               "  requires F<T> {\n"
+               "  return 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "int bar(T t)\n"
+               "  requires F<T>;",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "int bar(T t)\n"
+               "  requires F<T> {}",
+               Style);
+
   Style.RequiresClausePosition = FormatStyle::RCPS_SingleLine;
+  Style.IndentRequiresClause = false;
   verifyFormat("template <typename T> requires Foo<T> struct Bar {};\n"
                "template <typename T> requires Foo<T> void bar() {}\n"
                "template <typename T> void bar() requires Foo<T> {}\n"
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 256bbd49c424fa7..99798de43e70ffe 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -3360,6 +3360,15 @@ TEST_F(TokenAnnotatorTest, TypenameMacro) {
   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_Unknown);
 }
 
+TEST_F(TokenAnnotatorTest, GNULanguageStandard) {
+  auto Style = getGNUStyle();
+  EXPECT_EQ(Style.Standard, FormatStyle::LS_Latest);
+
+  auto Tokens = annotate("return 1 <=> 2;", Style);
+  ASSERT_EQ(Tokens.size(), 6u);
+  EXPECT_TOKEN(Tokens[2], tok::spaceship, TT_BinaryOperator);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang
diff --git a/clang/unittests/Rewrite/CMakeLists.txt b/clang/unittests/Rewrite/CMakeLists.txt
index 6594d8cc5897f34..3c5e2f8e5354bee 100644
--- a/clang/unittests/Rewrite/CMakeLists.txt
+++ b/clang/unittests/Rewrite/CMakeLists.txt
@@ -3,7 +3,6 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_clang_unittest(RewriteTests
-  RewriteBufferTest.cpp
   RewriterTest.cpp
   )
 clang_target_link_libraries(RewriteTests
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 6c283b68aa9656f..236574312338600 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -7302,7 +7302,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
   <tr class="open" id="1248">
     <td><a href="https://cplusplus.github.io/CWG/issues/1248.html">1248</a></td>
     <td>open</td>
-    <td>Updating Annex C to C99</td>
+    <td>Updating Annex C to C99 and C23</td>
     <td align="center">Not resolved</td>
   </tr>
   <tr id="1249">
@@ -12361,7 +12361,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2091.html">2091</a></td>
     <td>CD4</td>
     <td>Deducing reference non-type template arguments</td>
-    <td class="full" align="center">Clang 2.7</td>
+    <td class="full" align="center">Clang 10</td>
   </tr>
   <tr id="2092">
     <td><a href="https://cplusplus.github.io/CWG/issues/2092.html">2092</a></td>
@@ -15097,7 +15097,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2547.html">2547</a></td>
     <td>DRWP</td>
     <td>Defaulted comparison operator function for non-classes</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 20</td>
   </tr>
   <tr id="2548">
     <td><a href="https://cplusplus.github.io/CWG/issues/2548.html">2548</a></td>
@@ -15331,7 +15331,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2586.html">2586</a></td>
     <td>CD6</td>
     <td>Explicit object parameter for assignment and comparison</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 20</td>
   </tr>
   <tr class="open" id="2587">
     <td><a href="https://cplusplus.github.io/CWG/issues/2587.html">2587</a></td>
@@ -17318,13 +17318,37 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2917.html">2917</a></td>
     <td>open</td>
     <td>Disallow multiple <I>friend-type-specifier</I>s for a friend template</td>
-    <td align="center">Not resolved</td>
+    <td title="Clang 20 implements 2024-07-30 resolution" align="center">Not Resolved*</td>
   </tr>
   <tr class="open" id="2918">
     <td><a href="https://cplusplus.github.io/CWG/issues/2918.html">2918</a></td>
     <td>open</td>
     <td>Consideration of constraints for address of overloaded function</td>
     <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2919">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2919.html">2919</a></td>
+    <td>open</td>
+    <td>Conversion function candidates for initialization of const lvalue reference</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2920">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2920.html">2920</a></td>
+    <td>open</td>
+    <td>The <TT>template</TT> keyword for base classes</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2921">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2921.html">2921</a></td>
+    <td>open</td>
+    <td>Exporting redeclarations of entities not attached to a named module</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2922">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2922.html">2922</a></td>
+    <td>open</td>
+    <td>constexpr placement-new is too permissive</td>
+    <td align="center">Not resolved</td>
   </tr></table>
 
 </div>
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index a6ded8be3ae9e59..faee8b578b62423 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -202,7 +202,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Variadic friends</td>
   <td><a href="https://wg21.link/P2893R3">P2893R3</a></td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 20</td>
  </tr>
  <!-- Summer 2024 papers (St Louis) -->
  <tr>
diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index e041861edaf0b7b..9e66f77217ec6b9 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -576,8 +576,15 @@ struct Allocator {
     }
 
     AsanThread *t = GetCurrentThread();
-    void *allocated = allocator.Allocate(
-        GetAllocatorCache(&t->malloc_storage()), needed_size, 8);
+    void *allocated;
+    if (t) {
+      AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+      allocated = allocator.Allocate(cache, needed_size, 8);
+    } else {
+      SpinMutexLock l(&fallback_mutex);
+      AllocatorCache *cache = &fallback_allocator_cache;
+      allocated = allocator.Allocate(cache, needed_size, 8);
+    }
     if (UNLIKELY(!allocated)) {
       SetAllocatorOutOfMemory();
       if (AllocatorMayReturnNull())
diff --git a/compiler-rt/lib/asan/asan_allocator.h b/compiler-rt/lib/asan/asan_allocator.h
index b52cdeb73089239..db8dc3bebfc6206 100644
--- a/compiler-rt/lib/asan/asan_allocator.h
+++ b/compiler-rt/lib/asan/asan_allocator.h
@@ -189,7 +189,13 @@ const uptr kAllocatorSize = 0x40000000000ULL;  // 4T.
 typedef DefaultSizeClassMap SizeClassMap;
 #    endif  // SANITIZER_RISCV64
 #  else     // SANITIZER_FUCHSIA
+
+#    if SANITIZER_APPLE
+const uptr kAllocatorSpace = 0x600000000000ULL;
+#    else   // SANITIZER_APPLE
 const uptr kAllocatorSpace = ~(uptr)0;
+#    endif  // SANITIZER_APPLE
+
 #    if defined(__powerpc64__)
 const uptr kAllocatorSize  =  0x20000000000ULL;  // 2T.
 typedef DefaultSizeClassMap SizeClassMap;
diff --git a/compiler-rt/lib/asan/asan_flags.inc b/compiler-rt/lib/asan/asan_flags.inc
index fad1577d912a5e1..5e0ced9706e6647 100644
--- a/compiler-rt/lib/asan/asan_flags.inc
+++ b/compiler-rt/lib/asan/asan_flags.inc
@@ -36,11 +36,8 @@ ASAN_FLAG(int, max_redzone, 2048,
 ASAN_FLAG(
     bool, debug, false,
     "If set, prints some debugging information and does additional checks.")
-ASAN_FLAG(
-    int, report_globals, 1,
-    "Controls the way to handle globals (0 - don't detect buffer overflow on "
-    "globals, 1 - detect buffer overflow, 2 - print data about registered "
-    "globals).")
+ASAN_FLAG(bool, report_globals, true,
+          "If set, detect and report errors on globals .")
 ASAN_FLAG(bool, check_initialization_order, false,
           "If set, attempts to catch initialization order issues.")
 ASAN_FLAG(
diff --git a/compiler-rt/lib/asan/asan_globals.cpp b/compiler-rt/lib/asan/asan_globals.cpp
index c83b782cb85f896..a1211430b1268a0 100644
--- a/compiler-rt/lib/asan/asan_globals.cpp
+++ b/compiler-rt/lib/asan/asan_globals.cpp
@@ -22,6 +22,7 @@
 #include "asan_thread.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_dense_map.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_list.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
@@ -179,7 +180,7 @@ int GetGlobalsForAddress(uptr addr, Global *globals, u32 *reg_sites,
   int res = 0;
   for (const auto &l : list_of_all_globals) {
     const Global &g = *l.g;
-    if (flags()->report_globals >= 2)
+    if (UNLIKELY(common_flags()->verbosity >= 3))
       ReportGlobal(g, "Search");
     if (IsAddressNearGlobal(addr, g)) {
       internal_memcpy(&globals[res], &g, sizeof(g));
@@ -270,7 +271,7 @@ static inline bool UseODRIndicator(const Global *g) {
 // so we store the globals in a map.
 static void RegisterGlobal(const Global *g) SANITIZER_REQUIRES(mu_for_globals) {
   CHECK(AsanInited());
-  if (flags()->report_globals >= 2)
+  if (UNLIKELY(common_flags()->verbosity >= 3))
     ReportGlobal(*g, "Added");
   CHECK(flags()->report_globals);
   CHECK(AddrIsInMem(g->beg));
@@ -307,7 +308,7 @@ static void RegisterGlobal(const Global *g) SANITIZER_REQUIRES(mu_for_globals) {
 static void UnregisterGlobal(const Global *g)
     SANITIZER_REQUIRES(mu_for_globals) {
   CHECK(AsanInited());
-  if (flags()->report_globals >= 2)
+  if (UNLIKELY(common_flags()->verbosity >= 3))
     ReportGlobal(*g, "Removed");
   CHECK(flags()->report_globals);
   CHECK(AddrIsInMem(g->beg));
@@ -438,7 +439,7 @@ void __asan_register_globals(__asan_global *globals, uptr n) {
   }
   GlobalRegistrationSite site = {stack_id, &globals[0], &globals[n - 1]};
   global_registration_site_vector->push_back(site);
-  if (flags()->report_globals >= 2) {
+  if (UNLIKELY(common_flags()->verbosity >= 3)) {
     PRINT_CURRENT_STACK();
     Printf("=== ID %d; %p %p\n", stack_id, (void *)&globals[0],
            (void *)&globals[n - 1]);
@@ -497,9 +498,7 @@ void __asan_before_dynamic_init(const char *module_name) {
   Lock lock(&mu_for_globals);
   if (current_dynamic_init_module_name == module_name)
     return;
-  if (flags()->report_globals >= 3)
-    Printf("DynInitPoison module: %s\n", module_name);
-
+  VPrintf(2, "DynInitPoison module: %s\n", module_name);
   if (current_dynamic_init_module_name == nullptr) {
     // First call, poison all globals from other modules.
     DynInitGlobals().forEach([&](auto &kv) {
@@ -545,8 +544,7 @@ static void UnpoisonBeforeMain(void) {
       return;
     allow_after_dynamic_init = true;
   }
-  if (flags()->report_globals >= 3)
-    Printf("UnpoisonBeforeMain\n");
+  VPrintf(2, "UnpoisonBeforeMain\n");
   __asan_after_dynamic_init();
 }
 
@@ -570,8 +568,7 @@ void __asan_after_dynamic_init() {
   if (!current_dynamic_init_module_name)
     return;
 
-  if (flags()->report_globals >= 3)
-    Printf("DynInitUnpoison\n");
+  VPrintf(2, "DynInitUnpoison\n");
 
   DynInitGlobals().forEach([&](auto &kv) {
     UnpoisonDynamicGlobals(kv.second, /*mark_initialized=*/false);
diff --git a/compiler-rt/lib/asan/asan_poisoning.cpp b/compiler-rt/lib/asan/asan_poisoning.cpp
index 746ad61813c65dc..d600b1a0c241fa2 100644
--- a/compiler-rt/lib/asan/asan_poisoning.cpp
+++ b/compiler-rt/lib/asan/asan_poisoning.cpp
@@ -410,7 +410,7 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p,
                                                const void *new_mid_p) {
   if (!flags()->detect_container_overflow)
     return;
-  VPrintf(2, "contiguous_container: %p %p %p %p\n", beg_p, end_p, old_mid_p,
+  VPrintf(3, "contiguous_container: %p %p %p %p\n", beg_p, end_p, old_mid_p,
           new_mid_p);
   uptr storage_beg = reinterpret_cast<uptr>(beg_p);
   uptr storage_end = reinterpret_cast<uptr>(end_p);
@@ -479,7 +479,7 @@ void __sanitizer_annotate_double_ended_contiguous_container(
   if (!flags()->detect_container_overflow)
     return;
 
-  VPrintf(2, "contiguous_container: %p %p %p %p %p %p\n", storage_beg_p,
+  VPrintf(3, "contiguous_container: %p %p %p %p %p %p\n", storage_beg_p,
           storage_end_p, old_container_beg_p, old_container_end_p,
           new_container_beg_p, new_container_end_p);
 
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c
index ec4b08bfa89ac86..0dd397783b67f5a 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64.c
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c
@@ -45,7 +45,7 @@ _Bool __aarch64_have_lse_atomics
 #elif defined(__ANDROID__)
 #include "aarch64/hwcap.inc"
 #include "aarch64/lse_atomics/android.inc"
-#elif __has_include(<sys/auxv.h>)
+#elif defined(__linux__) && __has_include(<sys/auxv.h>)
 #include "aarch64/hwcap.inc"
 #include "aarch64/lse_atomics/getauxval.inc"
 #else
@@ -73,7 +73,7 @@ struct {
 #elif defined(__ANDROID__)
 #include "aarch64/fmv/mrs.inc"
 #include "aarch64/fmv/android.inc"
-#elif __has_include(<sys/auxv.h>)
+#elif defined(__linux__) && __has_include(<sys/auxv.h>)
 #include "aarch64/fmv/mrs.inc"
 #include "aarch64/fmv/getauxval.inc"
 #else
diff --git a/compiler-rt/lib/builtins/int_math.h b/compiler-rt/lib/builtins/int_math.h
index 74d3e311db5e710..08bfe922ffa13b4 100644
--- a/compiler-rt/lib/builtins/int_math.h
+++ b/compiler-rt/lib/builtins/int_math.h
@@ -65,9 +65,12 @@
 #define crt_copysign(x, y) __builtin_copysign((x), (y))
 #define crt_copysignf(x, y) __builtin_copysignf((x), (y))
 #define crt_copysignl(x, y) __builtin_copysignl((x), (y))
-#if __has_builtin(__builtin_copysignf128)
+// We define __has_builtin to always return 0 for GCC versions below 10,
+// but __builtin_copysignf128 is available since version 7.
+#if __has_builtin(__builtin_copysignf128) ||                                   \
+    (defined(__GNUC__) && __GNUC__ >= 7)
 #define crt_copysignf128(x, y) __builtin_copysignf128((x), (y))
-#elif __has_builtin(__builtin_copysignq) || (defined(__GNUC__) && __GNUC__ >= 7)
+#elif __has_builtin(__builtin_copysignq)
 #define crt_copysignf128(x, y) __builtin_copysignq((x), (y))
 #endif
 #endif
@@ -80,9 +83,11 @@
 #define crt_fabs(x) __builtin_fabs((x))
 #define crt_fabsf(x) __builtin_fabsf((x))
 #define crt_fabsl(x) __builtin_fabsl((x))
-#if __has_builtin(__builtin_fabsf128)
+// We define __has_builtin to always return 0 for GCC versions below 10,
+// but __builtin_fabsf128 is available since version 7.
+#if __has_builtin(__builtin_fabsf128) || (defined(__GNUC__) && __GNUC__ >= 7)
 #define crt_fabsf128(x) __builtin_fabsf128((x))
-#elif __has_builtin(__builtin_fabsq) || (defined(__GNUC__) && __GNUC__ >= 7)
+#elif __has_builtin(__builtin_fabsq)
 #define crt_fabsf128(x) __builtin_fabsq((x))
 #endif
 #endif
diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
index fe79e1908d60299..eefa0b326e51b54 100644
--- a/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
@@ -607,7 +607,13 @@ size_t PageSize() {
 }
 
 void SetThreadName(std::thread &thread, const std::string &name) {
-  // TODO ?
+  if (name.size() > 31)
+    name.resize(31);
+  zx_status_t s;
+  if ((s = zx_object_set_property(thread.native_handle(), ZX_PROP_NAME,
+                                  name.c_str(), name.size())) != ZX_OK)
+    Printf("SetThreadName for name %s failed: %s", name.c_str(),
+           zx_status_get_string(s));
 }
 
 } // namespace fuzzer
diff --git a/compiler-rt/lib/hwasan/scripts/hwasan_symbolize b/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
index 0408e0eb9993d06..d1b2857ccd8156f 100755
--- a/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
+++ b/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
@@ -109,6 +109,10 @@ class Symbolizer:
     self.__html = False
     self.__last_access_address = None
     self.__last_access_tag = None
+    self.__tag_dump = []
+    self.__tag_dump_match_idx = None
+    self.__matched_stack_uas = False
+    self.__offsets = []
 
   def enable_html(self, enable):
     self.__html = enable
@@ -311,6 +315,34 @@ class Symbolizer:
     if match:
       self.__last_access_tag = int(match.group(2), 16)
 
+  def process_tag_dump_line(self, line, ignore_tags=False):
+    m = re.match(r'.*?(0x[0-9a-f]+):' + '([ ]*[\[ ][0-9a-f][0-9a-f]\]?)' * 16, line)
+    if m is None:
+      return False
+    addr = m.group(1)
+    tags = m.group(*range(2, 18))
+    fault = [i for i, x in enumerate(tags) if '[' in x]
+    if fault:
+      self.__tag_dump_match_idx = len(self.__tag_dump) + fault[0]
+    self.__tag_dump.extend(int(x.strip(' [').rstrip('] '), 16) for x in tags)
+    return True
+
+  def finish_tag_dump(self):
+    if self.__matched_stack_uas or self.__tag_dump_match_idx is None:
+      return
+    for offset, size, local in sorted(self.__offsets, key=lambda x: abs(x[0])):
+      idx = self.__tag_dump_match_idx - offset // 16
+      if idx < 0 or idx > len(self.__tag_dump):
+        continue
+      if self.__tag_dump[idx] == self.__last_access_tag:
+        self.print('')
+        self.print('Potentially referenced stack object:')
+        if offset > 0:
+          self.print('  %d bytes after a variable "%s" in stack frame of function "%s"' % (offset - size, local[2], local[0]))
+        if offset < 0:
+          self.print('  %d bytes before a variable "%s" in stack frame of function "%s"' % (-offset, local[2], local[0]))
+        self.print('  at %s' % (local[1],))
+
   def process_stack_history(self, line, ignore_tags=False):
     if self.__last_access_address is None or self.__last_access_tag is None:
       return
@@ -336,16 +368,18 @@ class Symbolizer:
         size = local[4]
         if frame_offset is None or size is None:
           continue
-        obj_offset = (self.__last_access_address - fp - frame_offset) & fp_mask
-        if obj_offset >= size:
-          continue
+        obj_offset = (self.__last_access_address & fp_mask) - ((fp & fp_mask) + frame_offset)
         tag_offset = local[5]
         if not ignore_tags and (tag_offset is None or base_tag ^ tag_offset != self.__last_access_tag):
           continue
+        if obj_offset < 0 or obj_offset >= size:
+          self.__offsets.append((obj_offset, size, local))
+          continue
         self.print('')
         self.print('Potentially referenced stack object:')
         self.print('  %d bytes inside a variable "%s" in stack frame of function "%s"' % (obj_offset, local[2], local[0]))
         self.print('  at %s' % (local[1],))
+        self.__matched_stack_uas = True
       return True
     return False
 
@@ -456,9 +490,18 @@ def main():
       sys.exit(1)
     symbolizer.read_linkify(args.linkify)
 
+  tag_dump = False
   for line in sys.stdin:
     if sys.version_info.major < 3:
       line = line.decode('utf-8')
+    if tag_dump:
+      tag_dump = symbolizer.process_tag_dump_line(line)
+      if tag_dump:
+        continue
+      symbolizer.finish_tag_dump()
+    if 'Memory tags around the buggy address' in line:
+      tag_dump = True
+
     symbolizer.save_access_address(line)
     if symbolizer.process_stack_history(line, ignore_tags=args.ignore_tags):
       continue
diff --git a/compiler-rt/lib/msan/msan_allocator.cpp b/compiler-rt/lib/msan/msan_allocator.cpp
index f478b9979f2daad..d7d4967c949859f 100644
--- a/compiler-rt/lib/msan/msan_allocator.cpp
+++ b/compiler-rt/lib/msan/msan_allocator.cpp
@@ -199,8 +199,15 @@ static void *MsanAllocate(BufferedStackTrace *stack, uptr size, uptr alignment,
     ReportRssLimitExceeded(stack);
   }
   MsanThread *t = GetCurrentThread();
-  void *allocated = allocator.Allocate(GetAllocatorCache(&t->malloc_storage()),
-                                       size, alignment);
+  void *allocated;
+  if (t) {
+    AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+    allocated = allocator.Allocate(cache, size, alignment);
+  } else {
+    SpinMutexLock l(&fallback_mutex);
+    AllocatorCache *cache = &fallback_allocator_cache;
+    allocated = allocator.Allocate(cache, size, alignment);
+  }
   if (UNLIKELY(!allocated)) {
     SetAllocatorOutOfMemory();
     if (AllocatorMayReturnNull())
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
index 2428a8cd1479473..182dc8f26c88fd5 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
@@ -239,13 +239,15 @@ void RemoveANSIEscapeSequencesFromString(char *buffer);
 void Printf(const char *format, ...) FORMAT(1, 2);
 void Report(const char *format, ...) FORMAT(1, 2);
 void SetPrintfAndReportCallback(void (*callback)(const char *));
-#define VReport(level, ...)                                              \
-  do {                                                                   \
-    if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \
+#define VReport(level, ...)                     \
+  do {                                          \
+    if (UNLIKELY((uptr)Verbosity() >= (level))) \
+      Report(__VA_ARGS__);                      \
   } while (0)
-#define VPrintf(level, ...)                                              \
-  do {                                                                   \
-    if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \
+#define VPrintf(level, ...)                     \
+  do {                                          \
+    if (UNLIKELY((uptr)Verbosity() >= (level))) \
+      Printf(__VA_ARGS__);                      \
   } while (0)
 
 // Lock sanitizer error reporting and protects against nested errors.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 49c9dcbef358ff2..e09a4a8ae25fd8b 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -1255,6 +1255,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
   static const int PR_SET_VMA = 0x53564d41;
   static const int PR_SCHED_CORE = 62;
   static const int PR_SCHED_CORE_GET = 0;
+  static const int PR_GET_PDEATHSIG = 2;
   if (option == PR_SET_VMA && arg2 == 0UL) {
     char *name = (char *)arg5;
     COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
@@ -1270,7 +1271,9 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
   } else if (res != -1 && option == PR_SCHED_CORE &&
              arg2 == PR_SCHED_CORE_GET) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg5), sizeof(u64));
+  } else if (res != -1 && option == PR_GET_PDEATHSIG) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg2), sizeof(int));
   }
   return res;
 }
@@ -9980,7 +9983,7 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags);
   // If GRND_NONBLOCK is set in the flags, it is non blocking.
-  static const int grnd_nonblock = 1; 
+  static const int grnd_nonblock = 1;
   SSIZE_T n;
   if ((flags & grnd_nonblock))
     n = REAL(getrandom)(buf, buflen, flags);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
index ee293bbd68875c0..e2a66142e584e63 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -66,7 +66,7 @@ static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
 }
 
 static DTLS::DTV *DTLS_Find(uptr id) {
-  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
+  VReport(3, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
   static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
   DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
   if (!cur)
diff --git a/compiler-rt/lib/ubsan/ubsan_value.cpp b/compiler-rt/lib/ubsan/ubsan_value.cpp
index dc61e5b939d952b..6e88ebaf34d4be3 100644
--- a/compiler-rt/lib/ubsan/ubsan_value.cpp
+++ b/compiler-rt/lib/ubsan/ubsan_value.cpp
@@ -67,18 +67,21 @@ const char *__ubsan::getObjCClassName(ValueHandle Pointer) {
 
 SIntMax Value::getSIntValue() const {
   CHECK(getType().isSignedIntegerTy());
+  // Val was zero-extended to ValueHandle. Sign-extend from original width
+  // to SIntMax.
+  const unsigned ExtraBits =
+      sizeof(SIntMax) * 8 - getType().getIntegerBitCount();
   if (isInlineInt()) {
-    // Val was zero-extended to ValueHandle. Sign-extend from original width
-    // to SIntMax.
-    const unsigned ExtraBits =
-      sizeof(SIntMax) * 8 - getType().getIntegerBitWidth();
     return SIntMax(UIntMax(Val) << ExtraBits) >> ExtraBits;
   }
-  if (getType().getIntegerBitWidth() == 64)
-    return *reinterpret_cast<s64*>(Val);
+  if (getType().getIntegerBitWidth() == 64) {
+    return SIntMax(UIntMax(*reinterpret_cast<s64 *>(Val)) << ExtraBits) >>
+           ExtraBits;
+  }
 #if HAVE_INT128_T
   if (getType().getIntegerBitWidth() == 128)
-    return *reinterpret_cast<s128*>(Val);
+    return SIntMax(UIntMax(*reinterpret_cast<s128 *>(Val)) << ExtraBits) >>
+           ExtraBits;
 #else
   if (getType().getIntegerBitWidth() == 128)
     UNREACHABLE("libclang_rt.ubsan was built without __int128 support");
diff --git a/compiler-rt/lib/ubsan/ubsan_value.h b/compiler-rt/lib/ubsan/ubsan_value.h
index e0957276dd24197..430c9ea0dc8d150 100644
--- a/compiler-rt/lib/ubsan/ubsan_value.h
+++ b/compiler-rt/lib/ubsan/ubsan_value.h
@@ -103,6 +103,13 @@ class TypeDescriptor {
     /// representation is that of bitcasting the floating-point value to an
     /// integer type.
     TK_Float = 0x0001,
+    /// An _BitInt(N) type. Lowest bit is 1 for a signed value, 0 for an
+    /// unsigned value. Remaining bits are log_2(bit_width). The value
+    /// representation is the integer itself if it fits into a ValueHandle, and
+    /// a pointer to the integer otherwise. TypeName contains the true width
+    /// of the type for the signed _BitInt(N) type stored after zero bit after
+    /// TypeName as 32-bit unsigned integer.
+    TK_BitInt = 0x0002,
     /// Any other type. The value representation is unspecified.
     TK_Unknown = 0xffff
   };
@@ -113,10 +120,15 @@ class TypeDescriptor {
     return static_cast<Kind>(TypeKind);
   }
 
-  bool isIntegerTy() const { return getKind() == TK_Integer; }
+  bool isIntegerTy() const {
+    return getKind() == TK_Integer || getKind() == TK_BitInt;
+  }
+  bool isBitIntTy() const { return getKind() == TK_BitInt; }
+
   bool isSignedIntegerTy() const {
     return isIntegerTy() && (TypeInfo & 1);
   }
+  bool isSignedBitIntTy() const { return isBitIntTy() && (TypeInfo & 1); }
   bool isUnsignedIntegerTy() const {
     return isIntegerTy() && !(TypeInfo & 1);
   }
@@ -125,6 +137,25 @@ class TypeDescriptor {
     return 1 << (TypeInfo >> 1);
   }
 
+  const char *getBitIntBitCountPointer() const {
+    DCHECK(isBitIntTy());
+    DCHECK(isSignedBitIntTy());
+    // Scan Name for zero and return the next address
+    const char *p = getTypeName();
+    while (*p != '\0')
+      ++p;
+    // Return the next address
+    return p + 1;
+  }
+
+  unsigned getIntegerBitCount() const {
+    DCHECK(isIntegerTy());
+    if (isSignedBitIntTy())
+      return *reinterpret_cast<const u32 *>(getBitIntBitCountPointer());
+    else
+      return getIntegerBitWidth();
+  }
+
   bool isFloatTy() const { return getKind() == TK_Float; }
   unsigned getFloatBitWidth() const {
     CHECK(isFloatTy());
diff --git a/compiler-rt/test/asan/TestCases/Linux/initialization-nobug-lld.cpp b/compiler-rt/test/asan/TestCases/Linux/initialization-nobug-lld.cpp
index 5cec029811cbc84..ef82c7a29575ebd 100644
--- a/compiler-rt/test/asan/TestCases/Linux/initialization-nobug-lld.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/initialization-nobug-lld.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -O3 %S/../initialization-nobug.cpp %S/../Helpers/initialization-nobug-extra.cpp -fuse-ld=lld -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInit"
+// RUN: %clangxx_asan -O3 %S/../initialization-nobug.cpp %S/../Helpers/initialization-nobug-extra.cpp -fuse-ld=lld -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInit"
 
 // Same as initialization-nobug.cpp, but with lld we expect just one
 // `DynInitUnpoison` executed after `AfterDynamicInit` at the end.
diff --git a/compiler-rt/test/asan/TestCases/Linux/odr_indicator_unregister.cpp b/compiler-rt/test/asan/TestCases/Linux/odr_indicator_unregister.cpp
index 0f2ed6597154bba..b75f5be101ef8a4 100644
--- a/compiler-rt/test/asan/TestCases/Linux/odr_indicator_unregister.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/odr_indicator_unregister.cpp
@@ -4,7 +4,7 @@
 // RUN: %clangxx_asan -g -O0 -DSHARED_LIB -DSIZE=1 %s -fPIC -shared -o %t-so-1.so
 // RUN: %clangxx_asan -g -O0 -DSHARED_LIB -DSIZE=2 %s -fPIC -shared -o %t-so-2.so
 // RUN: %clangxx_asan -g -O0 %s %libdl -Wl,--export-dynamic -o %t
-// RUN: %env_asan_opts=report_globals=2:detect_odr_violation=1 %run %t 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=report_globals=1:detect_odr_violation=1:verbosity=3 %run %t 2>&1 | FileCheck %s
 
 // FIXME: Checks do not match on Android.
 // UNSUPPORTED: android
diff --git a/compiler-rt/test/asan/TestCases/Linux/odr_indicators.cpp b/compiler-rt/test/asan/TestCases/Linux/odr_indicators.cpp
index 8af3ec09be78c4f..f28a9f6d07386d5 100644
--- a/compiler-rt/test/asan/TestCases/Linux/odr_indicators.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/odr_indicators.cpp
@@ -1,8 +1,8 @@
 // RUN: %clangxx_asan -fno-sanitize-address-use-odr-indicator -fPIC %s -o %t
-// RUN: %env_asan_opts=report_globals=2 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,INDICATOR0
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,INDICATOR0
 
 // RUN: %clangxx_asan -fsanitize-address-use-odr-indicator -fPIC %s -o %t
-// RUN: %env_asan_opts=report_globals=2 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,INDICATOR1
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,INDICATOR1
 
 #include <stdio.h>
 
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_global_dead_strip.c b/compiler-rt/test/asan/TestCases/Windows/dll_global_dead_strip.c
index a0c96622efeea41..e5bd27bdf65fdf7 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_global_dead_strip.c
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_global_dead_strip.c
@@ -1,11 +1,11 @@
 // RUN: %clang_cl_asan %Od %p/dll_host.cpp %Fe%t
 //
 // RUN: %clang_cl_nocxx_asan %Gw %LD %Od %s %Fe%t.dll
-// RUN: %env_asan_opts=report_globals=2 %run %t %t.dll 2>&1 | FileCheck %s --check-prefix=NOSTRIP
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %t %t.dll 2>&1 | FileCheck %s --check-prefix=NOSTRIP
 // RUN: %clang_cl_nocxx_asan %Gw %LD -O2 %s %Fe%t.dll \
 // RUN:   %if target={{.*-windows-gnu}} %{ -Wl,--gc-sections %} \
 // RUN:   %else %{ -link -opt:ref %}
-// RUN: %env_asan_opts=report_globals=2 %run %t %t.dll 2>&1 | FileCheck %s --check-prefix=STRIP
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %t %t.dll 2>&1 | FileCheck %s --check-prefix=STRIP
 
 #include <stdio.h>
 
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_report_globals_symbolization_at_startup.cpp b/compiler-rt/test/asan/TestCases/Windows/dll_report_globals_symbolization_at_startup.cpp
index 06a632e6708b1e6..c74b66f2b43b3e6 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_report_globals_symbolization_at_startup.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_report_globals_symbolization_at_startup.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cl_asan %LD %Od -DDLL %s %Fe%t.dll \
 // RUN:   %if target={{.*-windows-gnu}} %{ -Wl,--out-implib,%t.lib %}
 // RUN: %clang_cl_asan %Od -DEXE %s %t.lib %Fe%te.exe
-// RUN: %env_asan_opts=report_globals=2 %run %te.exe 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %te.exe 2>&1 | FileCheck %s
 
 // FIXME: Currently, the MT runtime build crashes on startup due to dbghelp.dll
 // initialization failure.
diff --git a/compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c b/compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c
index 0e15120a46f776e..7f2405fdfc83644 100644
--- a/compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c
+++ b/compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c
@@ -1,9 +1,9 @@
 // RUN: %clang_cl_nocxx_asan %Gw %Od %s %Fe%t.exe
-// RUN: %env_asan_opts=report_globals=2 %t.exe 2>&1 | FileCheck %s --check-prefix=NOSTRIP
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %t.exe 2>&1 | FileCheck %s --check-prefix=NOSTRIP
 // RUN: %clang_cl_nocxx_asan %Gw -O2 %s %Fe%t.exe \
 // RUN:   %if target={{.*-windows-gnu}} %{ -Wl,--gc-sections %} \
 // RUN:   %else %{ -link -opt:ref %}
-// RUN: %env_asan_opts=report_globals=2 %t.exe 2>&1 | FileCheck %s --check-prefix=STRIP
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %t.exe 2>&1 | FileCheck %s --check-prefix=STRIP
 
 #include <stdio.h>
 int dead_global = 42;
diff --git a/compiler-rt/test/asan/TestCases/Windows/report_globals_vs_freelibrary.cpp b/compiler-rt/test/asan/TestCases/Windows/report_globals_vs_freelibrary.cpp
index 7cad3f39be1ec29..34ce18e146d6771 100644
--- a/compiler-rt/test/asan/TestCases/Windows/report_globals_vs_freelibrary.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/report_globals_vs_freelibrary.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cl_asan %LD %Od -DDLL %s %Fe%t.dll
 // RUN: %clang_cl_asan %Od -DEXE %s %Fe%te.exe
-// RUN: %env_asan_opts=report_globals=2 %run %te.exe %t.dll 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %te.exe %t.dll 2>&1 | FileCheck %s
 
 #include <windows.h>
 #include <stdio.h>
diff --git a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
new file mode 100644
index 000000000000000..369449303f3e082
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
@@ -0,0 +1,23 @@
+// RUN: %clangxx_asan %min_macos_deployment_target=10.11 -O0 %s %p/Helpers/initialization-bug-extra.cpp -o %t
+// RUN: %env_asan_opts=check_initialization_order=true:strict_init_order=true not %run %t 2>&1 | FileCheck %s
+
+// Do not test with optimization -- the error may be optimized away.
+
+// FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=186
+// XFAIL: target={{.*windows-msvc.*}}
+
+#include <stdio.h>
+
+extern int y;
+
+void __attribute__((constructor)) ctor() {
+  printf("%d\n", y);
+  // CHECK: AddressSanitizer: initialization-order-fiasco
+}
+
+int main() {
+  // ASan should have caused an exit before main runs.
+  printf("PASS\n");
+  // CHECK-NOT: PASS
+  return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/initialization-nobug.cpp b/compiler-rt/test/asan/TestCases/initialization-nobug.cpp
index f66d501124bc48b..61328b9de28ae6c 100644
--- a/compiler-rt/test/asan/TestCases/initialization-nobug.cpp
+++ b/compiler-rt/test/asan/TestCases/initialization-nobug.cpp
@@ -1,10 +1,10 @@
 // A collection of various initializers which shouldn't trip up initialization
 // order checking.  If successful, this will just return 0.
 
-// RUN: %clangxx_asan -O0 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
-// RUN: %clangxx_asan -O1 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
-// RUN: %clangxx_asan -O2 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
-// RUN: %clangxx_asan -O3 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
+// RUN: %clangxx_asan -O0 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
+// RUN: %clangxx_asan -O1 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
+// RUN: %clangxx_asan -O2 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
+// RUN: %clangxx_asan -O3 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
 
 // Simple access:
 // Make sure that accessing a global in the same TU is safe
diff --git a/compiler-rt/test/hwasan/TestCases/hwasan_symbolize_stack_overflow.cpp b/compiler-rt/test/hwasan/TestCases/hwasan_symbolize_stack_overflow.cpp
new file mode 100644
index 000000000000000..356f1da0818a3f4
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/hwasan_symbolize_stack_overflow.cpp
@@ -0,0 +1,30 @@
+// RUN: rm -rf %t; mkdir %t
+// RUN: %clang_hwasan -Wl,--build-id -g %s -o %t/hwasan_overflow
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow 16 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,AFTER0
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow 17 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,AFTER1
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow -1 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,BEFORE1
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow -17 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,BEFORE17
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow 1016 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,AFTER1000
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow -1000 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,BEFORE1000
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow 1000000 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=TOOFAR
+
+// REQUIRES: aarch64-target-arch
+
+#include <stdlib.h>
+
+static volatile char sink;
+
+int main(int argc, char **argv) {
+  volatile char x[16];
+  sink = x[atoi(argv[1])];
+  // TOOFAR-NOT: Potentially referenced stack object:
+  // CHECK: Potentially referenced stack object:
+  // AFTER0:   0 bytes after a variable "x" in stack frame of function "main"
+  // AFTER1:   1 bytes after a variable "x" in stack frame of function "main"
+  // BEFORE1:   1 bytes before a variable "x" in stack frame of function "main"
+  // BEFORE17:   17 bytes before a variable "x" in stack frame of function "main"
+  // BEFORE1000:   1000 bytes before a variable "x" in stack frame of function "main"
+  // AFTER1000:   1000 bytes after a variable "x" in stack frame of function "main"
+  // CHECK:   at {{.*}}hwasan_symbolize_stack_overflow.cpp:[[@LINE-10]]
+  return 0;
+}
diff --git a/compiler-rt/test/nsan/stable_sort.cpp b/compiler-rt/test/nsan/stable_sort.cpp
new file mode 100644
index 000000000000000..d280eedf0ebf7b6
--- /dev/null
+++ b/compiler-rt/test/nsan/stable_sort.cpp
@@ -0,0 +1,56 @@
+// RUN: %clangxx_nsan -O2 -g %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// Check compilation mode that is required to call memcpy/memmove.
+// RUN: %clangxx_nsan -fno-builtin -O2 -g %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// This tests a particularaly hard case of memory tracking. stable_sort does
+// conditional swaps of pairs of elements with mixed types (int/double).
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdio>
+#include <utility>
+#include <vector>
+
+extern "C" void __nsan_dump_shadow_mem(const char *addr, size_t size_bytes,
+                                       size_t bytes_per_line, size_t reserved);
+
+__attribute__((noinline)) void Run(std::vector<int> &indices,
+                                   std::vector<double> &values) {
+  const auto num_entries = indices.size();
+  std::vector<std::pair<int, double>> entries;
+  entries.reserve(num_entries);
+  for (size_t i = 0; i < num_entries; ++i)
+    entries.emplace_back(indices[i], values[i]);
+
+  __nsan_dump_shadow_mem((const char *)&entries[0].second, sizeof(double),
+                         sizeof(double), 0);
+  __nsan_dump_shadow_mem((const char *)&entries[1].second, sizeof(double),
+                         sizeof(double), 0);
+  // CHECK: {{.*}}: d0 d1 d2 d3 d4 d5 d6 d7 (1.02800000000000002487)
+  // CHECK-NEXT: {{.*}}: d0 d1 d2 d3 d4 d5 d6 d7 (7.95099999999999962341)
+  std::stable_sort(
+      entries.begin(), entries.end(),
+      [](const std::pair<int, double> &a, const std::pair<int, double> &b) {
+        return a.first < b.first;
+      });
+  __nsan_dump_shadow_mem((const char *)&entries[0].second, sizeof(double),
+                         sizeof(double), 0);
+  __nsan_dump_shadow_mem((const char *)&entries[1].second, sizeof(double),
+                         sizeof(double), 0);
+  // We make sure that the shadow values have been swapped correctly.
+  // CHECK-NEXT: {{.*}}: d0 d1 d2 d3 d4 d5 d6 d7 (7.95099999999999962341)
+  // CHECK-NEXT: {{.*}}: d0 d1 d2 d3 d4 d5 d6 d7 (1.02800000000000002487)
+}
+
+int main() {
+  std::vector<int> indices;
+  std::vector<double> values;
+  indices.push_back(75);
+  values.push_back(1.028);
+  indices.push_back(74);
+  values.push_back(7.951);
+  Run(indices, values);
+}
diff --git a/compiler-rt/test/nsan/swap.cpp b/compiler-rt/test/nsan/swap.cpp
new file mode 100644
index 000000000000000..0e1d9c8a880c02a
--- /dev/null
+++ b/compiler-rt/test/nsan/swap.cpp
@@ -0,0 +1,46 @@
+// RUN: %clangxx_nsan -O2 %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// RUN: %clangxx_nsan -fno-builtin -O2 %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// This verifies that shadow memory is tracked correcty across typed and
+// bitcasted swaps.
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <utility>
+
+extern "C" void __nsan_dump_shadow_mem(const char *addr, size_t size_bytes,
+                                       size_t bytes_per_line, size_t reserved);
+
+__attribute__((noinline)) void SwapFT(double &a, double &b) {
+  // LLVM typically optimizes this to an untyped swap (through i64) anyway.
+  std::swap(a, b);
+}
+
+__attribute__((noinline)) void SwapBitcasted(uint64_t &a, uint64_t &b) {
+  std::swap(a, b);
+}
+
+int main() {
+  double a = 1.0, b = 2.0;
+  __nsan_dump_shadow_mem((const char *)&a, sizeof(a), sizeof(a), 0);
+  __nsan_dump_shadow_mem((const char *)&b, sizeof(b), sizeof(b), 0);
+  SwapFT(a, b);
+  __nsan_dump_shadow_mem((const char *)&a, sizeof(a), sizeof(a), 0);
+  __nsan_dump_shadow_mem((const char *)&b, sizeof(b), sizeof(b), 0);
+  assert(a == 2.0 && b == 1.0);
+  SwapBitcasted(*reinterpret_cast<uint64_t *>(&a),
+                *reinterpret_cast<uint64_t *>(&b));
+  __nsan_dump_shadow_mem((const char *)&a, sizeof(a), sizeof(a), 0);
+  __nsan_dump_shadow_mem((const char *)&b, sizeof(b), sizeof(b), 0);
+  assert(a == 1.0 && b == 2.0);
+  // CHECK: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (1.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (2.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (2.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (1.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (1.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (2.0{{.*}}
+}
diff --git a/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c b/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c
index 3ed7c1894b6d106..d163a28d0e90ba6 100644
--- a/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c
+++ b/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c
@@ -4,12 +4,12 @@
 // mode to a new platform, but is not in and of itself a test of the profiling
 // runtime.
 
-// REQUIRES: darwin
+// REQUIRES: darwin, target={{arm64.*}}
 
 // Align counters and data to the maximum expected page size (16K).
 // RUN: %clang -g -o %t %s \
-// RUN:   -Wl,-sectalign,__DATA,__pcnts,0x1000 \
-// RUN:   -Wl,-sectalign,__DATA,__pdata,0x1000
+// RUN:   -Wl,-sectalign,__DATA,__pcnts,0x4000 \
+// RUN:   -Wl,-sectalign,__DATA,__pdata,0x4000
 
 // Create a 'profile' using mmap() and validate it.
 // RUN: %run %t create %t.tmpfile
@@ -24,7 +24,7 @@
 
 __attribute__((section("__DATA,__pcnts"))) int counters[] = {0xbad};
 extern int cnts_start __asm("section$start$__DATA$__pcnts");
-const size_t cnts_len = 0x1000;
+const size_t cnts_len = 0x4000;
 
 __attribute__((section("__DATA,__pdata"))) int data[] = {1, 2, 3};
 extern int data_start __asm("section$start$__DATA$__pdata");
@@ -131,12 +131,12 @@ int main(int argc, char **argv) {
     fprintf(stderr, "__pcnts is not page-aligned: 0x%lx.\n", cnts_start_int);
     return EXIT_FAILURE;
   }
-  if (data_start_int % 0x1000 != 0) {
+  if (data_start_int % 0x4000 != 0) {
     fprintf(stderr, "__pdata is not correctly aligned: 0x%lx.\n",
             data_start_int);
     return EXIT_FAILURE;
   }
-  if (cnts_start_int + 0x1000 != data_start_int) {
+  if (cnts_start_int + 0x4000 != data_start_int) {
     fprintf(stderr, "__pdata not ordered after __pcnts.\n");
     return EXIT_FAILURE;
   }
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/prctl.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/prctl.cpp
index d0be7f4fa87899d..cbff02d66efa78b 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/prctl.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/prctl.cpp
@@ -41,6 +41,14 @@ int main() {
     }
   }
 
+  int signum;
+  res = prctl(PR_GET_PDEATHSIG, reinterpret_cast<unsigned long>(&signum));
+  if (res < 0) {
+    assert(errno == EINVAL);
+  } else {
+    assert(signum == 0);
+  }
+
   char invname[81], vlname[] = "prctl";
   for (auto i = 0; i < sizeof(invname); i++) {
     invname[i] = 0x1e;
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
index 2d6e6c60f2437e5..c288e1d69baf92f 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
@@ -1,6 +1,6 @@
 // RUN: %clangxx %s -DBUILD_DSO -fPIC -shared -o %t.so
 // RUN: %clangxx --std=c++11 %s -o %t
-// RUN: %env_tool_opts=verbosity=2 %run %t 2>&1 | FileCheck %s
+// RUN: %env_tool_opts=verbosity=3 %run %t 2>&1 | FileCheck %s
 
 // Does not call __tls_get_addr
 // UNSUPPORTED: i386-linux
diff --git a/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c b/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c
new file mode 100644
index 000000000000000..2255d2d08c76616
--- /dev/null
+++ b/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c
@@ -0,0 +1,43 @@
+// REQUIRES: x86_64-target-arch
+// RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -O0 -fsanitize=alignment,array-bounds,bool,float-cast-overflow,implicit-integer-sign-change,implicit-signed-integer-truncation,implicit-unsigned-integer-truncation,integer-divide-by-zero,nonnull-attribute,null,nullability-arg,nullability-assign,nullability-return,pointer-overflow,returns-nonnull-attribute,shift-base,shift-exponent,signed-integer-overflow,unreachable,unsigned-integer-overflow,unsigned-shift-base,vla-bound %s -o %t1 && %run %t1 2>&1 | FileCheck %s
+
+#include <stdint.h>
+#include <stdio.h>
+
+// In this test there is an expectation of assignment of _BitInt not producing any output.
+uint32_t nullability_arg(_BitInt(37) *_Nonnull x)
+    __attribute__((no_sanitize("address")))
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) y = *(_BitInt(37) *)&x;
+  return (y > 0) ? y : 0;
+}
+
+// In this test there is an expectation of ubsan not triggeting on returning random address which is inside address space of the process.
+_BitInt(37) nonnull_attribute(__attribute__((nonnull)) _BitInt(37) * x)
+    __attribute__((no_sanitize("address")))
+    __attribute__((no_sanitize("memory"))) {
+  return *(_BitInt(37) *)&x;
+}
+
+// In this test there is an expectation of assignment of uint32_t from "invalid" _BitInt is not producing any output.
+uint32_t nullability_assign(_BitInt(7) * x)
+    __attribute__((no_sanitize("address")))
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(7) *_Nonnull y = x;
+  int32_t r = *(_BitInt(7) *)&y;
+  return (r > 0) ? r : 0;
+}
+
+// In those examples the file is expected to compile & run with no diagnostics
+// CHECK-NOT: runtime error:
+
+int main(int argc, char **argv) {
+  // clang-format off
+  uint64_t result =
+      1ULL +
+      nullability_arg((_BitInt(37) *)argc) +
+      ((uint64_t)nonnull_attribute((_BitInt(37) *)argc) & 0xFFFFFFFF) +
+      nullability_assign((_BitInt(7) *)argc);
+  // clang-format on
+  printf("%u\n", (uint32_t)(result & 0xFFFFFFFF));
+}
diff --git a/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c b/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
new file mode 100644
index 000000000000000..d7e2c6dc60aa0b4
--- /dev/null
+++ b/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
@@ -0,0 +1,172 @@
+// REQUIRES: x86_64-target-arch
+// REQUIRES: !windows
+// RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -O0 -fsanitize=array-bounds,float-cast-overflow,implicit-integer-sign-change,implicit-signed-integer-truncation,implicit-unsigned-integer-truncation,integer-divide-by-zero,pointer-overflow,shift-base,shift-exponent,signed-integer-overflow,unsigned-integer-overflow,unsigned-shift-base,vla-bound %s -o %t1 && %run %t1 2>&1 | FileCheck %s
+
+#include <stdint.h>
+#include <stdio.h>
+
+uint32_t float_divide_by_zero() {
+  float f = 1.0f / 0.0f;
+  _BitInt(37) r = (_BitInt(37))f;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:19: runtime error: inf is outside the range of representable values of type
+  return r;
+}
+
+uint32_t integer_divide_by_zero() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = 1 / 0;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:21: runtime error: division by zero
+  return x;
+}
+
+uint32_t implicit_unsigned_integer_truncation() {
+  unsigned _BitInt(37) x = 2U;
+  x += float_divide_by_zero();
+  x += integer_divide_by_zero();
+  x = x + 0xFFFFFFFFFFFFFFFFULL;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:9: runtime error: unsigned integer overflow:
+  uint32_t r = x & 0xFFFFFFFF;
+  return r;
+}
+
+uint32_t pointer_overflow() __attribute__((no_sanitize("address"))) {
+  _BitInt(37) *x = (_BitInt(37) *)1;
+  _BitInt(37) *y = x - 1;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:22: runtime error: pointer index expression with base
+  uint32_t r = *(_BitInt(37) *)&y;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:16: runtime error: implicit conversion from type
+  return r;
+}
+
+uint32_t vla_bound(_BitInt(37) x) {
+  _BitInt(37) a[x - 1];
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:17: runtime error: variable length array bound evaluates to non-positive value
+  return 0;
+}
+
+uint32_t unsigned_shift_base() {
+  unsigned _BitInt(37) x = ~0U << 1;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:32: runtime error: left shift of 4294967295 by 1 places cannot be represented in type
+  return x;
+}
+
+uint32_t array_bounds() {
+  _BitInt(37) x[4];
+  _BitInt(37) y = x[10];
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:19: runtime error: index 10 out of bounds for type
+  return (uint32_t)y;
+}
+
+uint32_t float_cast_overflow() {
+  float a = 100000000.0f;
+  _BitInt(7) b = (_BitInt(7))a;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:18: runtime error: 1e+08 is outside the range of representable values of type
+  return b;
+}
+
+uint32_t implicit_integer_sign_change(unsigned _BitInt(37) x) {
+  _BitInt(37) r = x;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:19: runtime error: implicit conversion from type '{{[^']+}}' of value
+  return r & 0xFFFFFFFF;
+}
+
+_BitInt(13) implicit_signed_integer_truncation() {
+#ifdef __SIZEOF_INT128__
+  _BitInt(73) x = (_BitInt(73)) ~((~0UL) >> 1);
+#else
+  uint32_t x = 0x7FFFFFFFUL;
+#endif
+  return x;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:10: runtime error: implicit conversion from type
+}
+
+_BitInt(37) shift_exponent() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = 1 << (-1);
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:21: runtime error: shift exponent -1 is negative
+  return x;
+}
+
+_BitInt(37) shift_base() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = (-1) << 1;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:24: runtime error: left shift of negative value -1
+  return x;
+}
+
+uint32_t negative_shift1(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(9) c = -2;
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t negative_shift2(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(17) c = -2;
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t negative_shift3(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(34) c = -2;
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t negative_shift4(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  int64_t c = -2;
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t negative_shift5(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+#ifdef __SIZEOF_INT128__
+  _BitInt(68) c = -2;
+#else
+  // We cannot check BitInt values > 64 without int128_t support
+  _BitInt(48) c = -2;
+#endif
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t unsigned_integer_overflow() __attribute__((no_sanitize("memory"))) {
+  unsigned _BitInt(37) x = ~0U;
+  ++x;
+  return x;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:10: runtime error: implicit conversion from type
+}
+
+// In this test no run-time overflow expected, so no diagnostics here, but should be a conversion error from the negative number on return.
+uint32_t signed_integer_overflow() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = (_BitInt(37)) ~((0x8FFFFFFFFFFFFFFFULL) >> 1);
+  --x;
+  return x;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:10: runtime error: implicit conversion from type
+}
+
+int main(int argc, char **argv) {
+  // clang-format off
+  uint64_t result =
+      1ULL +
+      implicit_unsigned_integer_truncation() +
+      pointer_overflow() +
+      vla_bound(argc) +
+      unsigned_shift_base() +
+      (uint32_t)array_bounds() +
+      float_cast_overflow() +
+      implicit_integer_sign_change((unsigned _BitInt(37))(argc - 2)) +
+      (uint64_t)implicit_signed_integer_truncation() +
+      shift_exponent() +
+      (uint32_t)shift_base() +
+      negative_shift1(5) +
+      negative_shift2(5) +
+      negative_shift3(5) +
+      negative_shift4(5) +
+      negative_shift5(5) +
+      unsigned_integer_overflow() +
+      signed_integer_overflow();
+  // clang-format on
+  printf("%u\n", (uint32_t)(result & 0xFFFFFFFF));
+}
diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index 4c7d88328a71e7e..e69876f07b63cbc 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -1056,8 +1056,9 @@ extern template class Integer<16>;
 extern template class Integer<32>;
 extern template class Integer<64>;
 using X87IntegerContainer =
-    Integer<80, true, 16, std::uint16_t, std::uint32_t, 128>;
-extern template class Integer<80, true, 16, std::uint16_t, std::uint32_t, 128>;
+    Integer<80, isHostLittleEndian, 16, std::uint16_t, std::uint32_t, 128>;
+extern template class Integer<80, isHostLittleEndian, 16, std::uint16_t,
+    std::uint32_t, 128>;
 extern template class Integer<128>;
 } // namespace Fortran::evaluate::value
 #endif // FORTRAN_EVALUATE_INTEGER_H_
diff --git a/flang/lib/Evaluate/integer.cpp b/flang/lib/Evaluate/integer.cpp
index b982a3a0796cc5c..17a38fa17a67ccb 100644
--- a/flang/lib/Evaluate/integer.cpp
+++ b/flang/lib/Evaluate/integer.cpp
@@ -14,7 +14,8 @@ template class Integer<8>;
 template class Integer<16>;
 template class Integer<32>;
 template class Integer<64>;
-template class Integer<80, true, 16, std::uint16_t, std::uint32_t, 128>;
+template class Integer<80, isHostLittleEndian, 16, std::uint16_t, std::uint32_t,
+    128>;
 template class Integer<128>;
 
 // Sanity checks against misconfiguration bugs
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 4d78f814f8ef2c9..6b3db619c1e2f1a 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1300,12 +1300,12 @@ std::optional<Expr<SomeType>> HollerithToBOZ(FoldingContext &context,
     const Expr<SomeType> &expr, const DynamicType &type) {
   if (std::optional<std::string> chValue{GetScalarConstantValue<Ascii>(expr)}) {
     // Pad on the right with spaces when short, truncate the right if long.
-    // TODO: big-endian targets
     auto bytes{static_cast<std::size_t>(
         ToInt64(type.MeasureSizeInBytes(context, false)).value())};
     BOZLiteralConstant bits{0};
     for (std::size_t j{0}; j < bytes; ++j) {
-      char ch{j >= chValue->size() ? ' ' : chValue->at(j)};
+      auto idx{isHostLittleEndian ? j : bytes - j - 1};
+      char ch{idx >= chValue->size() ? ' ' : chValue->at(idx)};
       BOZLiteralConstant chBOZ{static_cast<unsigned char>(ch)};
       bits = bits.IOR(chBOZ.SHIFTL(8 * j));
     }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index c5b809514c54c6c..7553e05b4706341 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -702,8 +702,53 @@ static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
   return reduction;
 }
 
-/// Given a reduction operation with an elemental mask, attempt to generate a
-/// do-loop to perform the operation inline.
+auto makeMinMaxInitValGenerator(bool isMax) {
+  return [isMax](fir::FirOpBuilder builder, mlir::Location loc,
+                 mlir::Type elementType) -> mlir::Value {
+    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
+      const llvm::fltSemantics &sem = ty.getFloatSemantics();
+      llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
+      return builder.createRealConstant(loc, elementType, limit);
+    }
+    unsigned bits = elementType.getIntOrFloatBitWidth();
+    int64_t limitInt =
+        isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
+              : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
+    return builder.createIntegerConstant(loc, elementType, limitInt);
+  };
+}
+
+mlir::Value generateMinMaxComparison(fir::FirOpBuilder builder,
+                                     mlir::Location loc, mlir::Value elem,
+                                     mlir::Value reduction, bool isMax) {
+  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
+    // For FP reductions we want the first smallest value to be used, that
+    // is not NaN. A OGL/OLT condition will usually work for this unless all
+    // the values are Nan or Inf. This follows the same logic as
+    // NumericCompare for Minloc/Maxlox in extrema.cpp.
+    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
+        loc,
+        isMax ? mlir::arith::CmpFPredicate::OGT
+              : mlir::arith::CmpFPredicate::OLT,
+        elem, reduction);
+    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
+    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
+    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
+    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
+    return builder.create<mlir::arith::CmpIOp>(
+        loc,
+        isMax ? mlir::arith::CmpIPredicate::sgt
+              : mlir::arith::CmpIPredicate::slt,
+        elem, reduction);
+  }
+  llvm_unreachable("unsupported type");
+}
+
+/// Given a reduction operation with an elemental/designate source, attempt to
+/// generate a do-loop to perform the operation inline.
 ///   %e = hlfir.elemental %shape unordered
 ///   %r = hlfir.count %e
 /// =>
@@ -712,17 +757,66 @@ static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
 ///     %c = <reduce count> %i
 ///     fir.result %c
 template <typename Op>
-class ReductionElementalConversion : public mlir::OpRewritePattern<Op> {
+class ReductionConversion : public mlir::OpRewritePattern<Op> {
 public:
   using mlir::OpRewritePattern<Op>::OpRewritePattern;
 
   llvm::LogicalResult
   matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
     mlir::Location loc = op.getLoc();
-    hlfir::ElementalOp elemental =
-        op.getMask().template getDefiningOp<hlfir::ElementalOp>();
-    if (!elemental || op.getDim())
-      return rewriter.notifyMatchFailure(op, "Did not find valid elemental");
+    // Select source and validate its arguments.
+    mlir::Value source;
+    bool valid = false;
+    if constexpr (std::is_same_v<Op, hlfir::AnyOp> ||
+                  std::is_same_v<Op, hlfir::AllOp> ||
+                  std::is_same_v<Op, hlfir::CountOp>) {
+      source = op.getMask();
+      valid = !op.getDim();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      source = op.getArray();
+      valid = !op.getDim() && !op.getMask();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                         std::is_same_v<Op, hlfir::MinlocOp>) {
+      source = op.getArray();
+      valid = !op.getDim() && !op.getMask() && !op.getBack();
+    }
+    if (!valid)
+      return rewriter.notifyMatchFailure(
+          op, "Currently does not accept optional arguments");
+
+    hlfir::ElementalOp elemental;
+    hlfir::DesignateOp designate;
+    mlir::Value shape;
+    if ((elemental = source.template getDefiningOp<hlfir::ElementalOp>())) {
+      shape = elemental.getOperand(0);
+    } else if ((designate =
+                    source.template getDefiningOp<hlfir::DesignateOp>())) {
+      shape = designate.getShape();
+    } else {
+      return rewriter.notifyMatchFailure(op, "Did not find valid argument");
+    }
+
+    auto inlineSource =
+        [elemental, &designate](
+            fir::FirOpBuilder builder, mlir::Location loc,
+            const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {
+      if (elemental) {
+        // Inline the elemental and get the value from it.
+        auto yield = inlineElementalOp(loc, builder, elemental, indices);
+        auto tmp = yield.getElementValue();
+        yield->erase();
+        return tmp;
+      }
+      if (designate) {
+        // Create a designator over designator, then load the reference.
+        auto resEntity = hlfir::Entity{designate.getResult()};
+        auto tmp = builder.create<hlfir::DesignateOp>(
+            loc, getVariableElementType(resEntity), designate, indices);
+        return builder.create<fir::LoadOp>(loc, tmp);
+      }
+      llvm_unreachable("unsupported type");
+    };
 
     fir::KindMapping kindMap =
         fir::getKindMapping(op->template getParentOfType<mlir::ModuleOp>());
@@ -732,47 +826,38 @@ class ReductionElementalConversion : public mlir::OpRewritePattern<Op> {
     GenBodyFn genBodyFn;
     if constexpr (std::is_same_v<Op, hlfir::AnyOp>) {
       init = builder.createIntegerConstant(loc, builder.getI1Type(), 0);
-      genBodyFn = [elemental](fir::FirOpBuilder builder, mlir::Location loc,
-                              mlir::Value reduction,
-                              const llvm::SmallVectorImpl<mlir::Value> &indices)
+      genBodyFn =
+          [inlineSource](fir::FirOpBuilder builder, mlir::Location loc,
+                         mlir::Value reduction,
+                         const llvm::SmallVectorImpl<mlir::Value> &indices)
           -> mlir::Value {
-        // Inline the elemental and get the condition from it.
-        auto yield = inlineElementalOp(loc, builder, elemental, indices);
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(), yield.getElementValue());
-        yield->erase();
-
         // Conditionally set the reduction variable.
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(), inlineSource(builder, loc, indices));
         return builder.create<mlir::arith::OrIOp>(loc, reduction, cond);
       };
     } else if constexpr (std::is_same_v<Op, hlfir::AllOp>) {
       init = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
-      genBodyFn = [elemental](fir::FirOpBuilder builder, mlir::Location loc,
-                              mlir::Value reduction,
-                              const llvm::SmallVectorImpl<mlir::Value> &indices)
+      genBodyFn =
+          [inlineSource](fir::FirOpBuilder builder, mlir::Location loc,
+                         mlir::Value reduction,
+                         const llvm::SmallVectorImpl<mlir::Value> &indices)
           -> mlir::Value {
-        // Inline the elemental and get the condition from it.
-        auto yield = inlineElementalOp(loc, builder, elemental, indices);
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(), yield.getElementValue());
-        yield->erase();
-
         // Conditionally set the reduction variable.
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(), inlineSource(builder, loc, indices));
         return builder.create<mlir::arith::AndIOp>(loc, reduction, cond);
       };
     } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
       init = builder.createIntegerConstant(loc, op.getType(), 0);
-      genBodyFn = [elemental](fir::FirOpBuilder builder, mlir::Location loc,
-                              mlir::Value reduction,
-                              const llvm::SmallVectorImpl<mlir::Value> &indices)
+      genBodyFn =
+          [inlineSource](fir::FirOpBuilder builder, mlir::Location loc,
+                         mlir::Value reduction,
+                         const llvm::SmallVectorImpl<mlir::Value> &indices)
           -> mlir::Value {
-        // Inline the elemental and get the condition from it.
-        auto yield = inlineElementalOp(loc, builder, elemental, indices);
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(), yield.getElementValue());
-        yield->erase();
-
         // Conditionally add one to the current value
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(), inlineSource(builder, loc, indices));
         mlir::Value one =
             builder.createIntegerConstant(loc, reduction.getType(), 1);
         mlir::Value add1 =
@@ -780,29 +865,49 @@ class ReductionElementalConversion : public mlir::OpRewritePattern<Op> {
         return builder.create<mlir::arith::SelectOp>(loc, cond, add1,
                                                      reduction);
       };
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                         std::is_same_v<Op, hlfir::MinlocOp>) {
+      // TODO: implement minloc/maxloc conversion.
+      return rewriter.notifyMatchFailure(
+          op, "Currently minloc/maxloc is not handled");
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      bool isMax = std::is_same_v<Op, hlfir::MaxvalOp>;
+      init = makeMinMaxInitValGenerator(isMax)(builder, loc, op.getType());
+      genBodyFn = [inlineSource,
+                   isMax](fir::FirOpBuilder builder, mlir::Location loc,
+                          mlir::Value reduction,
+                          const llvm::SmallVectorImpl<mlir::Value> &indices)
+          -> mlir::Value {
+        mlir::Value val = inlineSource(builder, loc, indices);
+        mlir::Value cmp =
+            generateMinMaxComparison(builder, loc, val, reduction, isMax);
+        return builder.create<mlir::arith::SelectOp>(loc, cmp, val, reduction);
+      };
     } else {
-      return mlir::failure();
+      llvm_unreachable("unsupported type");
     }
 
-    mlir::Value res = generateReductionLoop(builder, loc, init,
-                                            elemental.getOperand(0), genBodyFn);
+    mlir::Value res =
+        generateReductionLoop(builder, loc, init, shape, genBodyFn);
     if (res.getType() != op.getType())
       res = builder.create<fir::ConvertOp>(loc, op.getType(), res);
 
-    // Check if the op was the only user of the elemental (apart from a
-    // destroy), and remove it if so.
-    mlir::Operation::user_range elemUsers = elemental->getUsers();
-    hlfir::DestroyOp elemDestroy;
-    if (std::distance(elemUsers.begin(), elemUsers.end()) == 2) {
-      elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*elemUsers.begin());
-      if (!elemDestroy)
-        elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++elemUsers.begin());
+    // Check if the op was the only user of the source (apart from a destroy),
+    // and remove it if so.
+    mlir::Operation *sourceOp = source.getDefiningOp();
+    mlir::Operation::user_range srcUsers = sourceOp->getUsers();
+    hlfir::DestroyOp srcDestroy;
+    if (std::distance(srcUsers.begin(), srcUsers.end()) == 2) {
+      srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*srcUsers.begin());
+      if (!srcDestroy)
+        srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++srcUsers.begin());
     }
 
     rewriter.replaceOp(op, res);
-    if (elemDestroy) {
-      rewriter.eraseOp(elemDestroy);
-      rewriter.eraseOp(elemental);
+    if (srcDestroy) {
+      rewriter.eraseOp(srcDestroy);
+      rewriter.eraseOp(sourceOp);
     }
     return mlir::success();
   }
@@ -813,7 +918,7 @@ class ReductionElementalConversion : public mlir::OpRewritePattern<Op> {
 //  %e = hlfir.elemental %shape ({ ... })
 //  %m = hlfir.minloc %array mask %e
 template <typename Op>
-class MinMaxlocElementalConversion : public mlir::OpRewritePattern<Op> {
+class ReductionMaskConversion : public mlir::OpRewritePattern<Op> {
 public:
   using mlir::OpRewritePattern<Op>::OpRewritePattern;
 
@@ -848,19 +953,7 @@ class MinMaxlocElementalConversion : public mlir::OpRewritePattern<Op> {
         loc, fir::SequenceType::get(
                  rank, hlfir::getFortranElementType(mloc.getType())));
 
-    auto init = [isMax](fir::FirOpBuilder builder, mlir::Location loc,
-                        mlir::Type elementType) {
-      if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
-        const llvm::fltSemantics &sem = ty.getFloatSemantics();
-        llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
-        return builder.createRealConstant(loc, elementType, limit);
-      }
-      unsigned bits = elementType.getIntOrFloatBitWidth();
-      int64_t limitInt =
-          isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
-                : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
-      return builder.createIntegerConstant(loc, elementType, limitInt);
-    };
+    auto init = makeMinMaxInitValGenerator(isMax);
 
     auto genBodyOp =
         [&rank, &resultArr, &elemental, isMax](
@@ -900,33 +993,8 @@ class MinMaxlocElementalConversion : public mlir::OpRewritePattern<Op> {
       mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
 
       // Compare with the max reduction value
-      mlir::Value cmp;
-      if (mlir::isa<mlir::FloatType>(elementType)) {
-        // For FP reductions we want the first smallest value to be used, that
-        // is not NaN. A OGL/OLT condition will usually work for this unless all
-        // the values are Nan or Inf. This follows the same logic as
-        // NumericCompare for Minloc/Maxlox in extrema.cpp.
-        cmp = builder.create<mlir::arith::CmpFOp>(
-            loc,
-            isMax ? mlir::arith::CmpFPredicate::OGT
-                  : mlir::arith::CmpFPredicate::OLT,
-            elem, reduction);
-
-        mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
-            loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-        mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
-            loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-        cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-        cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
-      } else if (mlir::isa<mlir::IntegerType>(elementType)) {
-        cmp = builder.create<mlir::arith::CmpIOp>(
-            loc,
-            isMax ? mlir::arith::CmpIPredicate::sgt
-                  : mlir::arith::CmpIPredicate::slt,
-            elem, reduction);
-      } else {
-        llvm_unreachable("unsupported type");
-      }
+      mlir::Value cmp =
+          generateMinMaxComparison(builder, loc, elem, reduction, isMax);
 
       // The condition used for the loop is isFirst || <the condition above>.
       isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
@@ -1055,11 +1123,19 @@ class OptimizedBufferizationPass
     patterns.insert<ElementalAssignBufferization>(context);
     patterns.insert<BroadcastAssignBufferization>(context);
     patterns.insert<VariableAssignBufferization>(context);
-    patterns.insert<ReductionElementalConversion<hlfir::CountOp>>(context);
-    patterns.insert<ReductionElementalConversion<hlfir::AnyOp>>(context);
-    patterns.insert<ReductionElementalConversion<hlfir::AllOp>>(context);
-    patterns.insert<MinMaxlocElementalConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<MinMaxlocElementalConversion<hlfir::MaxlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
+    // TODO: implement basic minloc/maxloc conversion.
+    // patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
+    // patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
+    patterns.insert<ReductionMaskConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionMaskConversion<hlfir::MaxlocOp>>(context);
+    // TODO: implement masked minval/maxval conversion.
+    // patterns.insert<ReductionMaskConversion<hlfir::MaxvalOp>>(context);
+    // patterns.insert<ReductionMaskConversion<hlfir::MinvalOp>>(context);
 
     if (mlir::failed(mlir::applyPatternsAndFoldGreedily(
             getOperation(), std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index bdc2d9cd9c6c43f..a5a95138ac12819 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -457,10 +457,12 @@ StackArraysAnalysisWrapper::analyseFunction(mlir::Operation *func) {
     if (lattice)
       (void)point.join(*lattice);
   };
+
   func->walk([&](mlir::func::ReturnOp child) { joinOperationLattice(child); });
   func->walk([&](fir::UnreachableOp child) { joinOperationLattice(child); });
   func->walk(
       [&](mlir::omp::TerminatorOp child) { joinOperationLattice(child); });
+  func->walk([&](mlir::omp::YieldOp child) { joinOperationLattice(child); });
 
   llvm::DenseSet<mlir::Value> freedValues;
   point.appendFreedValues(freedValues);
diff --git a/flang/lib/Parser/provenance.cpp b/flang/lib/Parser/provenance.cpp
index e31038b09e40726..fe92aa7f64fb1db 100644
--- a/flang/lib/Parser/provenance.cpp
+++ b/flang/lib/Parser/provenance.cpp
@@ -248,7 +248,7 @@ static void EmitPrefix(llvm::raw_ostream &o, llvm::raw_ostream::Colors color,
 
 std::optional<ProvenanceRange> AllSources::GetInclusionInfo(
     const std::optional<ProvenanceRange> &range) const {
-  if (!range)
+  if (!range || !IsValid(range->start()))
     return std::nullopt;
   const Origin &origin{MapToOrigin(range->start())};
 
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 9fad1aa3dd0bf97..4708d51d3af4dd9 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -1645,7 +1645,8 @@ static void CheckPresent(evaluate::ActualArguments &arguments,
       } else {
         symbol = arg->GetAssumedTypeDummy();
       }
-      if (!symbol || !symbol->attrs().test(semantics::Attr::OPTIONAL)) {
+      if (!symbol ||
+          !symbol->GetUltimate().attrs().test(semantics::Attr::OPTIONAL)) {
         messages.Say(arg ? arg->sourceLocation() : messages.at(),
             "Argument of PRESENT() must be the name of a whole OPTIONAL dummy argument"_err_en_US);
       }
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index d635a7b8b7874fa..cc9f1cc7ed2691d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -2036,20 +2036,21 @@ void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
 void OmpAttributeVisitor::Post(const parser::Name &name) {
   auto *symbol{name.symbol};
   auto IsPrivatizable = [](const Symbol *sym) {
+    auto *misc{sym->detailsIf<MiscDetails>()};
     return !IsProcedure(*sym) && !IsNamedConstant(*sym) &&
         !sym->owner().IsDerivedType() &&
         sym->owner().kind() != Scope::Kind::ImpliedDos &&
         !sym->detailsIf<semantics::AssocEntityDetails>() &&
-        !sym->detailsIf<semantics::NamelistDetails>();
+        !sym->detailsIf<semantics::NamelistDetails>() &&
+        (!misc ||
+            (misc->kind() != MiscDetails::Kind::ComplexPartRe &&
+                misc->kind() != MiscDetails::Kind::ComplexPartIm &&
+                misc->kind() != MiscDetails::Kind::KindParamInquiry &&
+                misc->kind() != MiscDetails::Kind::LenParamInquiry &&
+                misc->kind() != MiscDetails::Kind::ConstructName));
   };
 
   if (symbol && !dirContext_.empty() && GetContext().withinConstruct) {
-    // Exclude construct-names
-    if (auto *details{symbol->detailsIf<semantics::MiscDetails>()}) {
-      if (details->kind() == semantics::MiscDetails::Kind::ConstructName) {
-        return;
-      }
-    }
     if (IsPrivatizable(symbol) && !IsObjectWithDSA(*symbol)) {
       // TODO: create a separate function to go through the rules for
       //       predetermined, explicitly determined, and implicitly
diff --git a/flang/runtime/CUDA/CMakeLists.txt b/flang/runtime/CUDA/CMakeLists.txt
index c9a20ebcc82e072..81055b2737c0f15 100644
--- a/flang/runtime/CUDA/CMakeLists.txt
+++ b/flang/runtime/CUDA/CMakeLists.txt
@@ -8,7 +8,12 @@
 
 include_directories(${CUDAToolkit_INCLUDE_DIRS})
 
-add_flang_library(CufRuntime
+# libCufRuntime depends on a certain version of CUDA. To be able to have
+# multiple build of this library with different CUDA version, the version is
+# added to the library name.
+set(CUFRT_LIBNAME CufRuntime_cuda_${CUDAToolkit_VERSION_MAJOR})
+
+add_flang_library(${CUFRT_LIBNAME}
   allocator.cpp
   descriptor.cpp
 )
@@ -19,7 +24,7 @@ else()
   set(CUDA_RT_TARGET CUDA::cudart_static)
 endif()
 
-target_link_libraries(CufRuntime
+target_link_libraries(${CUFRT_LIBNAME}
   PRIVATE
   FortranRuntime
   ${CUDA_RT_TARGET}
diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 562f8683b0ff7d8..0740ed509a07768 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -12,4 +12,4 @@ program main
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
-! ERROR: cuda-option.f90:8:19: error: expected end of statement
+! ERROR: cuda-option.f90:{{.*}}:{{.*}}: error: expected end of statement
diff --git a/flang/test/HLFIR/maxval-elemental.fir b/flang/test/HLFIR/maxval-elemental.fir
new file mode 100644
index 000000000000000..aa642253b08323a
--- /dev/null
+++ b/flang/test/HLFIR/maxval-elemental.fir
@@ -0,0 +1,95 @@
+// Test maxval inlining for both elemental and designate
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+// subroutine test(array)
+//   integer :: array(:), x
+//   x = maxval(abs(array))
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+  %c31_i32 = arith.constant 31 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg1: index):
+    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.shrsi %9, %c31_i32 : i32
+    %11 = arith.xori %9, %10 : i32
+    %12 = arith.subi %11, %10 : i32
+    hlfir.yield_element %12 : i32
+  }
+  %7 = hlfir.maxval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
+  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
+// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c-2147483648_i32) -> (i32) {
+// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
+// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi sgt, %[[V10]], %arg2 : i32
+// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
+// CHECK-NEXT:      fir.result %[[V12]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// subroutine test(array)
+//   real :: array(:), x
+//   x = maxval(array(3:6))
+// end subroutine test
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
+  %c4 = arith.constant 4 : index
+  %c1 = arith.constant 1 : index
+  %c6 = arith.constant 6 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+  %6 = hlfir.maxval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
+  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
+  return
+}
+
+// CHECK-LABEL: _QPtest_float
+// CHECK:       %cst = arith.constant 0xFF800000 : f32
+// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
+// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
+// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] : f32
+// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
+// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
+// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
+// CHECK-NEXT:      fir.result %[[V14]] : f32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
diff --git a/flang/test/HLFIR/minval-elemental.fir b/flang/test/HLFIR/minval-elemental.fir
new file mode 100644
index 000000000000000..8da1b1bdf515b9f
--- /dev/null
+++ b/flang/test/HLFIR/minval-elemental.fir
@@ -0,0 +1,95 @@
+// Test maxval inlining for both elemental and designate
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+// subroutine test(array)
+//   integer :: array(:), x
+//   x = minval(abs(array))
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+  %c31_i32 = arith.constant 31 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg1: index):
+    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.shrsi %9, %c31_i32 : i32
+    %11 = arith.xori %9, %10 : i32
+    %12 = arith.subi %11, %10 : i32
+    hlfir.yield_element %12 : i32
+  }
+  %7 = hlfir.minval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
+  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
+// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c2147483647_i32) -> (i32) {
+// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
+// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi slt, %[[V10]], %arg2 : i32
+// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
+// CHECK-NEXT:      fir.result %[[V12]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// subroutine test(array)
+//   real :: array(:), x
+//   x = minval(array(3:6))
+// end subroutine test
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
+  %c4 = arith.constant 4 : index
+  %c1 = arith.constant 1 : index
+  %c6 = arith.constant 6 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+  %6 = hlfir.minval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
+  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
+  return
+}
+
+// CHECK-LABEL: _QPtest_float
+// CHECK:       %cst = arith.constant 0x7F800000 : f32
+// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
+// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
+// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] : f32
+// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
+// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
+// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
+// CHECK-NEXT:      fir.result %[[V14]] : f32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
diff --git a/flang/test/Lower/ident.f90 b/flang/test/Lower/ident.f90
index c5201edbcf83f7b..2a6190b5f99182c 100644
--- a/flang/test/Lower/ident.f90
+++ b/flang/test/Lower/ident.f90
@@ -2,4 +2,4 @@
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
 ! CHECK: module attributes {
-! CHECK-SAME: llvm.ident = "flang version {{.+}}"
+! CHECK-SAME: llvm.ident = "{{.*}}flang version {{.+}}"
diff --git a/flang/test/Semantics/OpenMP/complex.f90 b/flang/test/Semantics/OpenMP/complex.f90
new file mode 100644
index 000000000000000..62336c7e6b31abd
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/complex.f90
@@ -0,0 +1,13 @@
+! RUN: %flang_fc1 -fopenmp -fsyntax-only %s
+
+! Check that using %re/%im inside 'parallel' doesn't cause syntax errors.
+subroutine test_complex_re_im
+  complex :: cc(4) = (1,2)
+  integer :: i
+
+  !$omp parallel do private(cc)
+    do i = 1, 4
+      print *, cc(i)%re, cc(i)%im
+    end do
+  !$omp end parallel do
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/present.f90 b/flang/test/Semantics/OpenMP/present.f90
new file mode 100644
index 000000000000000..31bdcc7a8c654ee
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/present.f90
@@ -0,0 +1,9 @@
+! RUN: %flang_fc1 -fopenmp -fsyntax-only %s
+
+! Check that using 'present' inside 'parallel' doesn't cause syntax errors.
+subroutine test_present(opt)
+  integer, optional :: opt
+  !$omp parallel
+    if (present(opt)) print *, "present"
+  !$omp end parallel
+end subroutine
diff --git a/flang/test/Semantics/data08.f90 b/flang/test/Semantics/data08.f90
index 7bd70637aabda78..7e12a71d117728e 100644
--- a/flang/test/Semantics/data08.f90
+++ b/flang/test/Semantics/data08.f90
@@ -1,12 +1,19 @@
-! RUN: %flang_fc1 -fdebug-dump-symbols -pedantic %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fdebug-dump-symbols -pedantic %s 2>&1 | FileCheck %s \
+! RUN:   --check-prefixes=%if system-aix %{"CHECK","BE"%} \
+! RUN:                    %else %{"CHECK","LE"%}
+
 ! CHECK: DATA statement value initializes 'jx' of type 'INTEGER(4)' with CHARACTER
 ! CHECK: DATA statement value initializes 'jy' of type 'INTEGER(4)' with CHARACTER
 ! CHECK: DATA statement value initializes 'jz' of type 'INTEGER(4)' with CHARACTER
 ! CHECK: DATA statement value initializes 'kx' of type 'INTEGER(8)' with CHARACTER
-! CHECK: jx (InDataStmt) size=4 offset=0: ObjectEntity type: INTEGER(4) init:1684234849_4
-! CHECK: jy (InDataStmt) size=4 offset=4: ObjectEntity type: INTEGER(4) init:543384161_4
-! CHECK: jz (InDataStmt) size=4 offset=8: ObjectEntity type: INTEGER(4) init:1684234849_4
-! CHECK: kx (InDataStmt) size=8 offset=16: ObjectEntity type: INTEGER(8) init:7523094288207667809_8
+! LE: jx (InDataStmt) size=4 offset=0: ObjectEntity type: INTEGER(4) init:1684234849_4
+! BE: jx (InDataStmt) size=4 offset=0: ObjectEntity type: INTEGER(4) init:1633837924_4
+! LE: jy (InDataStmt) size=4 offset=4: ObjectEntity type: INTEGER(4) init:543384161_4
+! BE: jy (InDataStmt) size=4 offset=4: ObjectEntity type: INTEGER(4) init:1633837856_4
+! LE: jz (InDataStmt) size=4 offset=8: ObjectEntity type: INTEGER(4) init:1684234849_4
+! BE: jz (InDataStmt) size=4 offset=8: ObjectEntity type: INTEGER(4) init:1633837924_4
+! LE: kx (InDataStmt) size=8 offset=16: ObjectEntity type: INTEGER(8) init:7523094288207667809_8
+! BE: kx (InDataStmt) size=8 offset=16: ObjectEntity type: INTEGER(8) init:7017280452245743464_8
 
 integer :: jx, jy, jz
 integer(8) :: kx
diff --git a/flang/test/Semantics/kinds04_q16.f90 b/flang/test/Semantics/kinds04_q16.f90
index 885ca8c8392b688..d440fa4d1402d27 100644
--- a/flang/test/Semantics/kinds04_q16.f90
+++ b/flang/test/Semantics/kinds04_q16.f90
@@ -1,4 +1,5 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1
+! RUN: not %flang_fc1 %s 2>%t.stderr
+! RUN: FileCheck %s --input-file=%t.stderr --check-prefixes=%if system-aix %{"PORTABILITY","WARNING","ERROR","AIX_WARNING"%} %else %{"PORTABILITY","WARNING","ERROR"%}
 ! C716 If both kind-param and exponent-letter appear, exponent-letter
 ! shall be E. (As an extension we also allow an exponent-letter which matches
 ! the kind-param)
@@ -15,10 +16,12 @@ subroutine s(var)
   !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   real :: realvar4 = 4.0D6_8
   !WARNING: Explicit kind parameter on real constant disagrees with exponent letter 'q'
+  !AIX_WARNING: underflow on REAL(10) to REAL(4) conversion
   real :: realvar5 = 4.0Q6_10
   !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   real :: realvar6 = 4.0Q6_16
   real :: realvar7 = 4.0E6_8
+  !AIX_WARNING: underflow on REAL(10) to REAL(4) conversion
   real :: realvar8 = 4.0E6_10
   real :: realvar9 = 4.0E6_16
   !ERROR: Unsupported REAL(KIND=32)
@@ -32,6 +35,7 @@ subroutine s(var)
   !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   double precision :: doublevar5 = 4.0Q6_16
   double precision :: doublevar6 = 4.0E6_8
+  !AIX_WARNING: underflow on REAL(10) to REAL(8) conversion
   double precision :: doublevar7 = 4.0E6_10
   double precision :: doublevar8 = 4.0E6_16
   !ERROR: Unsupported REAL(KIND=32)
diff --git a/flang/test/Transforms/stack-arrays-hlfir.f90 b/flang/test/Transforms/stack-arrays-hlfir.f90
index 50261b3078466d0..296edbaf35417e8 100644
--- a/flang/test/Transforms/stack-arrays-hlfir.f90
+++ b/flang/test/Transforms/stack-arrays-hlfir.f90
@@ -53,3 +53,31 @@ end subroutine omp_temp_array
 ! CHECK-NEXT:    }
 ! CHECK:         return
 ! CHECK-NEXT:  }
+
+subroutine omp_target_wsloop
+  implicit none
+  integer (8) :: lV, i
+  integer (8), dimension (2) :: iaVS
+
+  lV = 202
+
+  !$omp target teams distribute
+  do i = 1, 10
+    iaVS = [lV, lV]
+  end do
+  !$omp end target teams distribute
+end subroutine omp_target_wsloop
+! CHECK-LABEL: func.func @_QPomp_target_wsloop{{.*}} {
+! CHECK:         omp.target {{.*}} {
+! CHECK-NOT:       fir.allocmem
+! CHECK-NOT:       fir.freemem
+! CHECK:           fir.alloca !fir.array<2xi64>
+! CHECK:         omp.teams {
+! CHECK:         omp.distribute {
+! CHECK:         omp.loop_nest {{.*}} {
+! CHECK-NOT:       fir.allocmem
+! CHECK-NOT:       fir.freemem
+! CHECK:           omp.yield
+! CHECK-NEXT:    }
+! CHECK:         return
+! CHECK-NEXT:  }
diff --git a/flang/unittests/Runtime/CUDA/CMakeLists.txt b/flang/unittests/Runtime/CUDA/CMakeLists.txt
index 14b5c788719b870..ed0caece3d15db4 100644
--- a/flang/unittests/Runtime/CUDA/CMakeLists.txt
+++ b/flang/unittests/Runtime/CUDA/CMakeLists.txt
@@ -6,7 +6,7 @@ add_flang_unittest(FlangCufRuntimeTests
 
 target_link_libraries(FlangCufRuntimeTests
   PRIVATE
-  CufRuntime
+  CufRuntime_cuda_${CUDAToolkit_VERSION_MAJOR}
   FortranRuntime
 )
 
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
index f237e2ea1b9545e..920c5b206b0fefa 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
@@ -115,7 +115,7 @@ void print_results(Benchmark *b) {
   cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
 
   LIBC_NAMESPACE::printf(
-      "%-20s |%8ld |%8ld |%8ld |%11d |%14ld %2s |%9ld |%9d |\n",
+      "%-24s |%8ld |%8ld |%8ld |%11d |%14ld %2s |%9ld |%9d |\n",
       b->get_test_name().data(), result.cycles, result.min, result.max,
       result.total_iterations, result.total_time, time_unit,
       static_cast<uint64_t>(result.standard_deviation), num_threads);
@@ -127,7 +127,7 @@ void print_header() {
                          benchmarks[0]->get_suite_name().data());
   LIBC_NAMESPACE::printf("%s", RESET);
   cpp::string titles =
-      "Benchmark            |  Cycles |     Min |     Max | "
+      "Benchmark                |  Cycles |     Min |     Max | "
       "Iterations | Time / Iteration |   Stddev |  Threads |\n";
   LIBC_NAMESPACE::printf(titles.data());
 
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.h b/libc/benchmarks/gpu/LibcGpuBenchmark.h
index 830e6f9e89a7439..f2cfbfbfdcdf0db 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.h
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.h
@@ -146,10 +146,8 @@ template <typename T> class MathPerf {
       cpp::numeric_limits<StorageType>::max();
 
 public:
-  typedef T Func(T);
-
   template <size_t N = 1>
-  static uint64_t run_throughput_in_range(Func f, int min_exp, int max_exp) {
+  static uint64_t run_throughput_in_range(T f(T), int min_exp, int max_exp) {
     cpp::array<T, N> inputs;
     for (size_t i = 0; i < N; ++i)
       inputs[i] = get_rand_input<T>(min_exp, max_exp);
@@ -158,6 +156,23 @@ template <typename T> class MathPerf {
 
     return total_time / N;
   }
+
+  // Throughput benchmarking for functions that take 2 inputs.
+  template <size_t N = 1>
+  static uint64_t run_throughput_in_range(T f(T, T), int arg1_min_exp,
+                                          int arg1_max_exp, int arg2_min_exp,
+                                          int arg2_max_exp) {
+    cpp::array<T, N> inputs1;
+    cpp::array<T, N> inputs2;
+    for (size_t i = 0; i < N; ++i) {
+      inputs1[i] = get_rand_input<T>(arg1_min_exp, arg1_max_exp);
+      inputs2[i] = get_rand_input<T>(arg2_min_exp, arg2_max_exp);
+    }
+
+    uint64_t total_time = LIBC_NAMESPACE::throughput(f, inputs1, inputs2);
+
+    return total_time / N;
+  }
 };
 
 } // namespace benchmarks
diff --git a/libc/benchmarks/gpu/src/math/CMakeLists.txt b/libc/benchmarks/gpu/src/math/CMakeLists.txt
index 335da5ad71cf882..6870c024490166d 100644
--- a/libc/benchmarks/gpu/src/math/CMakeLists.txt
+++ b/libc/benchmarks/gpu/src/math/CMakeLists.txt
@@ -43,3 +43,22 @@ add_benchmark(
   LOADER_ARGS
     --threads 64
 )
+
+add_benchmark(
+  atan2_benchmark
+  SUITE
+    libc-gpu-math-benchmarks
+  SRCS
+    atan2_benchmark.cpp
+  DEPENDS
+    libc.src.math.atan2
+    libc.src.stdlib.srand
+    libc.src.stdlib.rand
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.CPP.bit
+    libc.src.__support.CPP.array
+  COMPILE_OPTIONS
+    ${math_benchmark_flags}
+  LOADER_ARGS
+    --threads 64
+)
diff --git a/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp b/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp
new file mode 100644
index 000000000000000..3bb5b0cc6788ca1
--- /dev/null
+++ b/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp
@@ -0,0 +1,47 @@
+#include "benchmarks/gpu/LibcGpuBenchmark.h"
+
+#include "src/math/atan2.h"
+#include "src/stdlib/rand.h"
+
+#ifdef NVPTX_MATH_FOUND
+#include "src/math/nvptx/declarations.h"
+#endif
+
+#ifdef AMDGPU_MATH_FOUND
+#include "src/math/amdgpu/declarations.h"
+#endif
+
+#define BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, N)                      \
+  []() {                                                                       \
+    return LIBC_NAMESPACE::benchmarks::MathPerf<T>::run_throughput_in_range<   \
+        N>(Func, MIN_EXP, MAX_EXP, MIN_EXP, MAX_EXP);                          \
+  }
+
+#define BENCH(T, Name, Func, MIN_EXP, MAX_EXP)                                 \
+  SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_1,                   \
+                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 1));    \
+  SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_128,                 \
+                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 128));  \
+  SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_1024,                \
+                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 1024)); \
+  SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_4096,                \
+                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 4096))
+
+BENCH(double, Atan2, LIBC_NAMESPACE::atan2, -1023, 1023);
+BENCH(double, Atan2TwoPi, LIBC_NAMESPACE::atan2, -10, 3);
+BENCH(double, Atan2TwoPow30, LIBC_NAMESPACE::atan2, 0, 30);
+BENCH(double, Atan2Large, LIBC_NAMESPACE::atan2, 30, 1000);
+
+#ifdef NVPTX_MATH_FOUND
+BENCH(double, NvAtan2, LIBC_NAMESPACE::__nv_atan2, -1023, 1023);
+BENCH(double, NvAtan2TwoPi, LIBC_NAMESPACE::__nv_atan2, -10, 3);
+BENCH(double, NvAtan2TwoPow30, LIBC_NAMESPACE::__nv_atan2, 0, 30);
+BENCH(double, NvAtan2Large, LIBC_NAMESPACE::__nv_atan2, 30, 1000);
+#endif
+
+#ifdef AMDGPU_MATH_FOUND
+BENCH(double, AmdAtan2, LIBC_NAMESPACE::__ocml_atan2_f64, -1023, 1023);
+BENCH(double, AmdAtan2TwoPi, LIBC_NAMESPACE::__ocml_atan2_f64, -10, 3);
+BENCH(double, AmdAtan2TwoPow30, LIBC_NAMESPACE::__ocml_atan2_f64, 0, 30);
+BENCH(double, AmdAtan2Large, LIBC_NAMESPACE::__ocml_atan2_f64, 30, 1000);
+#endif
diff --git a/libc/benchmarks/gpu/timing/amdgpu/timing.h b/libc/benchmarks/gpu/timing/amdgpu/timing.h
index d5c3df27b7de605..4cf7e9838add344 100644
--- a/libc/benchmarks/gpu/timing/amdgpu/timing.h
+++ b/libc/benchmarks/gpu/timing/amdgpu/timing.h
@@ -130,6 +130,31 @@ throughput(F f, const cpp::array<T, N> &inputs) {
   return stop - start;
 }
 
+// Provides throughput benchmarking for 2 arguments (e.g. atan2())
+template <typename F, typename T, size_t N>
+[[gnu::noinline]] static LIBC_INLINE uint64_t throughput(
+    F f, const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {
+  asm("" ::"v"(&inputs1), "v"(&inputs2));
+
+  gpu::memory_fence();
+  uint64_t start = gpu::processor_clock();
+
+  asm("" ::"s"(start));
+
+  for (size_t i = 0; i < inputs1.size(); i++) {
+    auto result = f(inputs1[i], inputs2[i]);
+
+    asm("" ::"v"(result));
+  }
+
+  uint64_t stop = gpu::processor_clock();
+  asm("" ::"s"(stop));
+  gpu::memory_fence();
+
+  // Return the time elapsed.
+  return stop - start;
+}
+
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_UTILS_GPU_TIMING_AMDGPU
diff --git a/libc/benchmarks/gpu/timing/nvptx/timing.h b/libc/benchmarks/gpu/timing/nvptx/timing.h
index 637986abd9092da..ece7d9a6c5396c2 100644
--- a/libc/benchmarks/gpu/timing/nvptx/timing.h
+++ b/libc/benchmarks/gpu/timing/nvptx/timing.h
@@ -121,6 +121,32 @@ throughput(F f, const cpp::array<T, N> &inputs) {
   // Return the time elapsed.
   return stop - start;
 }
+
+// Provides throughput benchmarking for 2 arguments (e.g. atan2())
+template <typename F, typename T, size_t N>
+[[gnu::noinline]] static LIBC_INLINE uint64_t throughput(
+    F f, const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {
+  asm("" ::"r"(&inputs1), "r"(&inputs2));
+
+  gpu::memory_fence();
+  uint64_t start = gpu::processor_clock();
+
+  asm("" ::"llr"(start));
+
+  uint64_t result;
+  for (size_t i = 0; i < inputs1.size(); i++) {
+    result = f(inputs1[i], inputs2[i]);
+    asm("" ::"r"(result));
+  }
+
+  uint64_t stop = gpu::processor_clock();
+  gpu::memory_fence();
+  asm("" ::"r"(stop));
+  volatile auto output = result;
+
+  // Return the time elapsed.
+  return stop - start;
+}
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_UTILS_GPU_TIMING_NVPTX
diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index b2644d2ebf386cf..bbae3298fae615e 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -248,6 +248,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.atan
     libc.src.math.atan2
     libc.src.math.atan2f
+    libc.src.math.atan2l
     libc.src.math.atanf
     libc.src.math.atanh
     libc.src.math.atanhf
@@ -424,13 +425,15 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.rint
     libc.src.math.rintf
     libc.src.math.rintl
-    libc.src.math.round
     libc.src.math.roundeven
     libc.src.math.roundevenf
     libc.src.math.roundevenl
+    libc.src.math.round
     libc.src.math.roundf
+    libc.src.math.roundl
     libc.src.math.scalbln
     libc.src.math.scalblnf
+    libc.src.math.scalblnl
     libc.src.math.scalbn
     libc.src.math.scalbnf
     libc.src.math.scalbnl
@@ -458,6 +461,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.tgammaf
     libc.src.math.totalorder
     libc.src.math.totalorderf
+    libc.src.math.totalorderl
     libc.src.math.totalordermag
     libc.src.math.totalordermagf
     libc.src.math.totalordermagl
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 2b6eb61782a6326..4e3ae7f801f4a08 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -420,19 +420,21 @@ add_header_macro(
     .llvm-libc-types.posix_spawn_file_actions_t
 )
 
-add_gen_header(
+add_header_macro(
   link
-  DEF_FILE link.h.def
-  GEN_HDR link.h
+  ../libc/newhdrgen/yaml/link.yaml
+  link.h.def
+  link.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.link_macros
 )
 
-add_gen_header(
+add_header_macro(
   elf
-  DEF_FILE elf.h.def
-  GEN_HDR elf.h
+  ../libc/newhdrgen/yaml/elf.yaml
+  elf.h.def
+  elf.h
   DEPENDS
     .llvm-libc-macros.elf_macros
 )
diff --git a/libc/newhdrgen/yaml/math.yaml b/libc/newhdrgen/yaml/math.yaml
index f8b105514271c70..04b6a073deace04 100644
--- a/libc/newhdrgen/yaml/math.yaml
+++ b/libc/newhdrgen/yaml/math.yaml
@@ -37,6 +37,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: atan2
+    standards:
+      - stdc
+    return_type: double
+    arguments:
+      - type: double
+      - type: double
   - name: atan2f
     standards:
       - stdc
@@ -44,6 +51,13 @@ functions:
     arguments:
       - type: float
       - type: float
+  - name: atan2l
+    standards:
+      - stdc
+    return_type: long double
+    arguments:
+      - type: long double
+      - type: long double
   - name: atanf
     standards:
       - stdc
diff --git a/libc/newhdrgen/yaml/string.yaml b/libc/newhdrgen/yaml/string.yaml
index 387ea9ec4e1422a..1d6e64bfb9cf60a 100644
--- a/libc/newhdrgen/yaml/string.yaml
+++ b/libc/newhdrgen/yaml/string.yaml
@@ -181,7 +181,7 @@ functions:
       - BSDExtensions
     return_type: size_t
     arguments:
-      - type: const char *__restrict
+      - type: char *__restrict
       - type: const char *__restrict
       - type: size_t
   - name: strlcpy
@@ -189,7 +189,7 @@ functions:
       - BSDExtensions
     return_type: size_t
     arguments:
-      - type: const char *__restrict
+      - type: char *__restrict
       - type: const char *__restrict
       - type: size_t
   - name: strlen
diff --git a/libc/spec/bsd_ext.td b/libc/spec/bsd_ext.td
index 4d33313521735e7..2b91324e36db9aa 100644
--- a/libc/spec/bsd_ext.td
+++ b/libc/spec/bsd_ext.td
@@ -20,12 +20,12 @@ def BsdExtensions : StandardSpec<"BSDExtensions"> {
         FunctionSpec<
             "strlcat",
             RetValSpec<SizeTType>,
-            [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
+            [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
         >,
         FunctionSpec<
             "strlcpy",
             RetValSpec<SizeTType>,
-            [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
+            [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
         >,
         FunctionSpec<
             "strsep",
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 8497acbfd62f0d3..118dcce829be232 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -715,6 +715,7 @@ def StdC : StandardSpec<"stdc"> {
 
           FunctionSpec<"atan2", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"atan2f", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"atan2l", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"acoshf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"asinhf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
diff --git a/libc/src/__support/FPUtil/FEnvImpl.h b/libc/src/__support/FPUtil/FEnvImpl.h
index 4bb31a6f4c426cd..1c5a1108ff9e0be 100644
--- a/libc/src/__support/FPUtil/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/FEnvImpl.h
@@ -31,7 +31,7 @@
 #include "x86_64/FEnvImpl.h"
 #elif defined(LIBC_TARGET_ARCH_IS_ARM) && defined(__ARM_FP)
 #include "arm/FEnvImpl.h"
-#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
+#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) && defined(__riscv_flen)
 #include "riscv/FEnvImpl.h"
 #else
 
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 132511a536366cc..3cba34fc2493223 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -55,6 +55,7 @@ add_math_entrypoint_object(atanf)
 
 add_math_entrypoint_object(atan2)
 add_math_entrypoint_object(atan2f)
+add_math_entrypoint_object(atan2l)
 
 add_math_entrypoint_object(atanh)
 add_math_entrypoint_object(atanhf)
diff --git a/libc/src/math/atan2l.h b/libc/src/math/atan2l.h
new file mode 100644
index 000000000000000..134d570d72e1f52
--- /dev/null
+++ b/libc/src/math/atan2l.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for atan2l ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ATAN2L_H
+#define LLVM_LIBC_SRC_MATH_ATAN2L_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+long double atan2l(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ATAN2L_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 745bd65e1d75b51..350072f4b9649d0 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -4230,6 +4230,17 @@ add_entrypoint_object(
     libc.src.__support.macros.optimization
 )
 
+add_entrypoint_object(
+  atan2l
+  SRCS
+    atan2l.cpp
+  HDRS
+    ../atan2l.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    .atan2
+)
 
 add_entrypoint_object(
   scalbln
diff --git a/libc/src/math/generic/atan2l.cpp b/libc/src/math/generic/atan2l.cpp
new file mode 100644
index 000000000000000..47a2e985f380c55
--- /dev/null
+++ b/libc/src/math/generic/atan2l.cpp
@@ -0,0 +1,26 @@
+//===-- Extended-precision atan2 function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/atan2l.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/math/atan2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+// TODO: Implement this for extended precision.
+LLVM_LIBC_FUNCTION(long double, atan2l, (long double y, long double x)) {
+#if defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
+  return static_cast<long double>(
+      atan2(static_cast<double>(y), static_cast<double>(x)));
+#else
+#error "Extended precision is not yet supported"
+#endif
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/startup/gpu/amdgpu/start.cpp b/libc/startup/gpu/amdgpu/start.cpp
index 5aaa7e938d27924..f09541b0d558081 100644
--- a/libc/startup/gpu/amdgpu/start.cpp
+++ b/libc/startup/gpu/amdgpu/start.cpp
@@ -41,7 +41,9 @@ static void call_fini_array_callbacks() {
 
 } // namespace LIBC_NAMESPACE_DECL
 
-extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel]] void
+extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel,
+             clang::amdgpu_flat_work_group_size(1, 1),
+             clang::amdgpu_max_num_work_groups(1)]] void
 _begin(int argc, char **argv, char **env) {
   __atomic_store_n(&LIBC_NAMESPACE::app.env_ptr,
                    reinterpret_cast<uintptr_t *>(env), __ATOMIC_RELAXED);
@@ -60,7 +62,9 @@ _start(int argc, char **argv, char **envp, int *ret) {
   __atomic_fetch_or(ret, main(argc, argv, envp), __ATOMIC_RELAXED);
 }
 
-extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel]] void
+extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel,
+             clang::amdgpu_flat_work_group_size(1, 1),
+             clang::amdgpu_max_num_work_groups(1)]] void
 _end(int retval) {
   // Only a single thread should call `exit` here, the rest should gracefully
   // return from the kernel. This is so only one thread calls the destructors
diff --git a/libc/utils/mathtools/worst_case.sollya b/libc/utils/mathtools/worst_case.sollya
new file mode 100644
index 000000000000000..3a8d11b3da44d51
--- /dev/null
+++ b/libc/utils/mathtools/worst_case.sollya
@@ -0,0 +1,80 @@
+// Implement WorstCase functions to compute the worst case for x mod C, with
+// the exponent of x ranges from emin to emax, and precision of x is p.
+// Adapted to Sollya from the Maple function in 
+//   J-M. Muller, "Elementary Functions", 3rd ed, Section 11.3.2.
+//
+// Some examples:
+//
+// 1) Worst case for trig range reduction fast passes:
+//
+// Single precision
+// > WorstCase(24, -6, 32, pi/32, 128); 
+// numbermin :  10741887
+// expmin    :  7
+// Worst case:  0x1.47d0fep30
+// numberofdigits :  -32.888
+//
+// Double precision
+// > WorstCase(53, -8, 32, pi/128, 256);
+// numbermin :  6411027962775774
+// expmin    :  -53
+// Worst case:  0x1.6c6cbc45dc8dep-1
+// numberofdigits :  -66.4867
+//
+// 2) Worst case for exponential function range reduction:
+//
+// Single precision
+// > WorstCase(24, 1, 8, log(2), 128);
+// numbermin :  2907270
+// expmin    :  -22
+// Worst case:  0x1.62e43p-1
+// numberofdigits :  -28.9678
+//
+// Double precision
+// > WorstCase(53, 0, 11, log(2), 128);
+// numbermin :  7804143460206699
+// expmin    :  -51
+// Worst case:  0x1.bb9d3beb8c86bp1
+// numberofdigits :  -57.4931
+//
+verbosity=0;
+procedure WorstCase(p, emin, emax, C, ndigits) {
+    epsilonmin = 12345.0;
+    Digits = ndigits;
+
+    powerofBoverC = 2^(emin - p) / C;
+    for e from emin - p + 1 to emax - p + 1 do {
+        powerofBoverC = 2 * powerofBoverC;
+        a = floor(powerofBoverC);
+        Plast = a;
+        r = round(1/(powerofBoverC - a), ndigits, RN);
+        a = floor(r);
+        Qlast = 1;
+        Q = a;
+        P = Plast * a + 1;
+        while (Q < 2^p - 1) do {
+            r = round(1/(r - a), ndigits, RN);
+            a = floor(r);
+            NewQ = Q * a + Qlast;
+            NewP = P * a + Plast;
+            Qlast = Q;
+            Plast = P;
+            Q = NewQ;
+            P = NewP;
+        };
+        epsilon = C * abs(Plast - Qlast * powerofBoverC);
+        if (epsilon < epsilonmin) then {
+            epsilonmin = epsilon;
+            numbermin = Qlast;
+            expmin = e;
+        };
+    };
+    display=decimal!;
+    print("numbermin : ", numbermin);
+    print("expmin    : ", expmin);
+    display=hexadecimal!;
+    print("Worst case: ", numbermin * 2^expmin);
+    display=decimal!;
+    ell = round(log2(epsilonmin), ndigits, RN);
+    print("numberofdigits : ", ell);
+};
diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index b319067394099a2..fe9f4c1973cdb43 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -62,7 +62,7 @@ Deprecations and Removals
   ``_LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS`` macro that was used to re-enable this extension will be
   ignored in LLVM 20.
 
-- TODO: The ``_LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST`` macro will no longer have an effect.
+- The ``_LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST`` macro no longer has any effect. ``std::allocator<const T>`` is not supported as an extension anymore, please migrate any code that uses e.g. ``std::vector<const T>`` to be standards conforming.
 
 Upcoming Deprecations and Removals
 ----------------------------------
diff --git a/libcxx/docs/Status/Cxx17.rst b/libcxx/docs/Status/Cxx17.rst
index 3f1f2071300c803..9263bd7c4af0c8e 100644
--- a/libcxx/docs/Status/Cxx17.rst
+++ b/libcxx/docs/Status/Cxx17.rst
@@ -41,7 +41,7 @@ Paper Status
 .. note::
 
    .. [#note-P0067] P0067: ``std::(to|from)_chars`` for integrals has been available since version 7.0. ``std::to_chars`` for ``float`` and ``double`` since version 14.0 ``std::to_chars`` for ``long double`` uses the implementation for ``double``.
-   .. [#note-P0226] P0226: Progress is tracked `here <https://https://libcxx.llvm.org/Status/SpecialMath.html>`_.
+   .. [#note-P0226] P0226: Progress is tracked `here <https://libcxx.llvm.org/Status/SpecialMath.html>`_.
    .. [#note-P0607] P0607: The parts of P0607 that are not done are the ``<regex>`` bits.
    .. [#note-P0154] P0154: The required macros are only implemented as of clang 19.
    .. [#note-P0452] P0452: The changes to ``std::transform_inclusive_scan`` and ``std::transform_exclusive_scan`` have not yet been implemented.
diff --git a/libcxx/include/__math/traits.h b/libcxx/include/__math/traits.h
index 27ec52ecef022e9..35c283cc9e21cef 100644
--- a/libcxx/include/__math/traits.h
+++ b/libcxx/include/__math/traits.h
@@ -79,20 +79,22 @@ _LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf
   return false;
 }
 
-#ifdef _LIBCPP_PREFERRED_OVERLOAD
 _LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(float __x) _NOEXCEPT {
   return __builtin_isinf(__x);
 }
 
-_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD bool
-isinf(double __x) _NOEXCEPT {
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI
+#ifdef _LIBCPP_PREFERRED_OVERLOAD
+_LIBCPP_PREFERRED_OVERLOAD
+#endif
+    bool
+    isinf(double __x) _NOEXCEPT {
   return __builtin_isinf(__x);
 }
 
 _LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(long double __x) _NOEXCEPT {
   return __builtin_isinf(__x);
 }
-#endif
 
 // isnan
 
@@ -106,20 +108,22 @@ _LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan
   return false;
 }
 
-#ifdef _LIBCPP_PREFERRED_OVERLOAD
 _LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(float __x) _NOEXCEPT {
   return __builtin_isnan(__x);
 }
 
-_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD bool
-isnan(double __x) _NOEXCEPT {
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI
+#ifdef _LIBCPP_PREFERRED_OVERLOAD
+_LIBCPP_PREFERRED_OVERLOAD
+#endif
+    bool
+    isnan(double __x) _NOEXCEPT {
   return __builtin_isnan(__x);
 }
 
 _LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(long double __x) _NOEXCEPT {
   return __builtin_isnan(__x);
 }
-#endif
 
 // isnormal
 
diff --git a/libcxx/include/__memory/allocator.h b/libcxx/include/__memory/allocator.h
index ae1f549626ee49e..0dbdc41d3c3d148 100644
--- a/libcxx/include/__memory/allocator.h
+++ b/libcxx/include/__memory/allocator.h
@@ -47,22 +47,6 @@ class _LIBCPP_TEMPLATE_VIS allocator<void> {
     typedef allocator<_Up> other;
   };
 };
-
-// TODO(LLVM 20): Remove the escape hatch
-#  ifdef _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
-template <>
-class _LIBCPP_TEMPLATE_VIS allocator<const void> {
-public:
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* pointer;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* const_pointer;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void value_type;
-
-  template <class _Up>
-  struct _LIBCPP_DEPRECATED_IN_CXX17 rebind {
-    typedef allocator<_Up> other;
-  };
-};
-#  endif // _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
 #endif   // _LIBCPP_STD_VER <= 17
 
 // This class provides a non-trivial default constructor to the class that derives from it
@@ -171,85 +155,6 @@ class _LIBCPP_TEMPLATE_VIS allocator : private __non_trivial_if<!is_void<_Tp>::v
 #endif
 };
 
-// TODO(LLVM 20): Remove the escape hatch
-#ifdef _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
-template <class _Tp>
-class _LIBCPP_TEMPLATE_VIS allocator<const _Tp>
-    : private __non_trivial_if<!is_void<_Tp>::value, allocator<const _Tp> > {
-  static_assert(!is_volatile<_Tp>::value, "std::allocator does not support volatile types");
-
-public:
-  typedef size_t size_type;
-  typedef ptrdiff_t difference_type;
-  typedef const _Tp value_type;
-  typedef true_type propagate_on_container_move_assignment;
-#  if _LIBCPP_STD_VER <= 23 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_ALLOCATOR_MEMBERS)
-  _LIBCPP_DEPRECATED_IN_CXX23 typedef true_type is_always_equal;
-#  endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator() _NOEXCEPT = default;
-
-  template <class _Up>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator(const allocator<_Up>&) _NOEXCEPT {}
-
-  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const _Tp* allocate(size_t __n) {
-    if (__n > allocator_traits<allocator>::max_size(*this))
-      __throw_bad_array_new_length();
-    if (__libcpp_is_constant_evaluated()) {
-      return static_cast<const _Tp*>(::operator new(__n * sizeof(_Tp)));
-    } else {
-      return static_cast<const _Tp*>(std::__libcpp_allocate(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)));
-    }
-  }
-
-#  if _LIBCPP_STD_VER >= 23
-  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr allocation_result<const _Tp*> allocate_at_least(size_t __n) {
-    return {allocate(__n), __n};
-  }
-#  endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void deallocate(const _Tp* __p, size_t __n) {
-    if (__libcpp_is_constant_evaluated()) {
-      ::operator delete(const_cast<_Tp*>(__p));
-    } else {
-      std::__libcpp_deallocate((void*)const_cast<_Tp*>(__p), __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
-    }
-  }
-
-  // C++20 Removed members
-#  if _LIBCPP_STD_VER <= 17
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* pointer;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& reference;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& const_reference;
-
-  template <class _Up>
-  struct _LIBCPP_DEPRECATED_IN_CXX17 rebind {
-    typedef allocator<_Up> other;
-  };
-
-  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI const_pointer address(const_reference __x) const _NOEXCEPT {
-    return std::addressof(__x);
-  }
-
-  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX17 const _Tp* allocate(size_t __n, const void*) {
-    return allocate(__n);
-  }
-
-  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
-    return size_type(~0) / sizeof(_Tp);
-  }
-
-  template <class _Up, class... _Args>
-  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI void construct(_Up* __p, _Args&&... __args) {
-    ::new ((void*)__p) _Up(std::forward<_Args>(__args)...);
-  }
-
-  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI void destroy(pointer __p) { __p->~_Tp(); }
-#  endif
-};
-#endif // _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
-
 template <class _Tp, class _Up>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
 operator==(const allocator<_Tp>&, const allocator<_Up>&) _NOEXCEPT {
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index 7475ef5cf85def4..72db3a266fdd4d9 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -562,17 +562,13 @@ struct __allocator_has_trivial_copy_construct<allocator<_Type>, _Type> : true_ty
 
 template <class _Alloc,
           class _In,
-          class _RawTypeIn = __remove_const_t<_In>,
           class _Out,
-          __enable_if_t<
-              // using _RawTypeIn because of the allocator<T const> extension
-              is_trivially_copy_constructible<_RawTypeIn>::value && is_trivially_copy_assignable<_RawTypeIn>::value &&
-                  is_same<__remove_const_t<_In>, __remove_const_t<_Out> >::value &&
-                  __allocator_has_trivial_copy_construct<_Alloc, _RawTypeIn>::value,
-              int> = 0>
+          __enable_if_t<is_trivially_copy_constructible<_In>::value && is_trivially_copy_assignable<_In>::value &&
+                            is_same<__remove_const_t<_In>, __remove_const_t<_Out> >::value &&
+                            __allocator_has_trivial_copy_construct<_Alloc, _In>::value,
+                        int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Out*
 __uninitialized_allocator_copy_impl(_Alloc&, _In* __first1, _In* __last1, _Out* __first2) {
-  // TODO: Remove the const_cast once we drop support for std::allocator<T const>
   if (__libcpp_is_constant_evaluated()) {
     while (__first1 != __last1) {
       std::__construct_at(std::__to_address(__first2), *__first1);
@@ -581,7 +577,7 @@ __uninitialized_allocator_copy_impl(_Alloc&, _In* __first1, _In* __last1, _Out*
     }
     return __first2;
   } else {
-    return std::copy(__first1, __last1, const_cast<_RawTypeIn*>(__first2));
+    return std::copy(__first1, __last1, __first2);
   }
 }
 
@@ -642,7 +638,7 @@ __uninitialized_allocator_relocate(_Alloc& __alloc, _Tp* __first, _Tp* __last, _
     __guard.__complete();
     std::__allocator_destroy(__alloc, __first, __last);
   } else {
-    __builtin_memcpy(const_cast<__remove_const_t<_Tp>*>(__result), __first, sizeof(_Tp) * (__last - __first));
+    __builtin_memcpy(__result, __first, sizeof(_Tp) * (__last - __first));
   }
 }
 
diff --git a/libcxx/include/span b/libcxx/include/span
index 60d76d830f0f318..da631cdc3f90e6b 100644
--- a/libcxx/include/span
+++ b/libcxx/include/span
@@ -206,10 +206,10 @@ struct __is_std_span<span<_Tp, _Sz>> : true_type {};
 
 template <class _Range, class _ElementType>
 concept __span_compatible_range =
+    !__is_std_span<remove_cvref_t<_Range>>::value &&                //
     ranges::contiguous_range<_Range> &&                             //
     ranges::sized_range<_Range> &&                                  //
     (ranges::borrowed_range<_Range> || is_const_v<_ElementType>) && //
-    !__is_std_span<remove_cvref_t<_Range>>::value &&                //
     !__is_std_array<remove_cvref_t<_Range>>::value &&               //
     !is_array_v<remove_cvref_t<_Range>> &&                          //
     is_convertible_v<remove_reference_t<ranges::range_reference_t<_Range>> (*)[], _ElementType (*)[]>;
diff --git a/libcxx/src/filesystem/path.cpp b/libcxx/src/filesystem/path.cpp
index b2019521377ede5..58742442bae6b71 100644
--- a/libcxx/src/filesystem/path.cpp
+++ b/libcxx/src/filesystem/path.cpp
@@ -267,7 +267,7 @@ path path::lexically_relative(const path& base) const {
   // Find the first mismatching element
   auto PP     = PathParser::CreateBegin(__pn_);
   auto PPBase = PathParser::CreateBegin(base.__pn_);
-  while (PP && PPBase && PP.State_ == PPBase.State_ && *PP == *PPBase) {
+  while (PP && PPBase && PP.State_ == PPBase.State_ && (*PP == *PPBase || PP.inRootDir())) {
     ++PP;
     ++PPBase;
   }
@@ -368,7 +368,8 @@ size_t hash_value(const path& __p) noexcept {
   size_t hash_value = 0;
   hash<string_view_t> hasher;
   while (PP) {
-    hash_value = __hash_combine(hash_value, hasher(*PP));
+    string_view_t Part = PP.inRootDir() ? PATHSTR("/") : *PP;
+    hash_value         = __hash_combine(hash_value, hasher(Part));
     ++PP;
   }
   return hash_value;
diff --git a/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp b/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
index 69c46f00fb65dc7..87f4783e12973e5 100644
--- a/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
+++ b/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
@@ -76,7 +76,7 @@ int main(int, char**) {
   test_allocations(64, 64);
   // Size being a multiple of alignment also needs to be supported.
   test_allocations(64, 32);
-  // When aligned allocation is implemented using posix_memalign,
+  // When aligned allocation is implemented using aligned_alloc,
   // that function requires a minimum alignment of sizeof(void*).
   // Check that we can also create overaligned allocations with
   // an alignment argument less than sizeof(void*).
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/copy.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/copy.pass.cpp
index 28f13e122ddc5e1..d3990fd60a459aa 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/copy.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -14,58 +15,101 @@
 #include <span>
 #include <cassert>
 #include <string>
+#include <utility>
 
 #include "test_macros.h"
 
-template <typename T>
-constexpr bool doCopy(const T &rhs)
-{
-    ASSERT_NOEXCEPT(T{rhs});
-    T lhs{rhs};
-    return lhs.data() == rhs.data()
-     &&    lhs.size() == rhs.size();
-}
+template <class T>
+constexpr void test() {
+  ASSERT_NOEXCEPT(std::span<T>(std::declval<std::span<T> const&>()));
+  ASSERT_NOEXCEPT(std::span<T>{std::declval<std::span<T> const&>()});
 
-struct A{};
-
-template <typename T>
-void testCV ()
-{
-    int  arr[] = {1,2,3};
-    assert((doCopy(std::span<T>  ()          )));
-    assert((doCopy(std::span<T,0>()          )));
-    assert((doCopy(std::span<T>  (&arr[0], 1))));
-    assert((doCopy(std::span<T,1>(&arr[0], 1))));
-    assert((doCopy(std::span<T>  (&arr[0], 2))));
-    assert((doCopy(std::span<T,2>(&arr[0], 2))));
+  // dynamic_extent
+  {
+    std::span<T> x;
+    std::span<T> copy(x);
+    assert(copy.data() == x.data());
+    assert(copy.size() == x.size());
+  }
+  {
+    T array[3] = {};
+    std::span<T> x(array, 3);
+    std::span<T> copy(x);
+    assert(copy.data() == array);
+    assert(copy.size() == 3);
+  }
+  {
+    T array[3] = {};
+    std::span<T> x(array, 2);
+    std::span<T> copy(x);
+    assert(copy.data() == array);
+    assert(copy.size() == 2);
+  }
+
+  // static extent
+  {
+    std::span<T, 0> x;
+    std::span<T, 0> copy(x);
+    assert(copy.data() == x.data());
+    assert(copy.size() == x.size());
+  }
+  {
+    T array[3] = {};
+    std::span<T, 3> x(array);
+    std::span<T, 3> copy(x);
+    assert(copy.data() == array);
+    assert(copy.size() == 3);
+  }
+  {
+    T array[2] = {};
+    std::span<T, 2> x(array);
+    std::span<T, 2> copy(x);
+    assert(copy.data() == array);
+    assert(copy.size() == 2);
+  }
 }
 
+struct Foo {};
+
+constexpr bool test_all() {
+  test<int>();
+  test<const int>();
+  test<volatile int>();
+  test<const volatile int>();
 
-int main(int, char**)
-{
-    constexpr int carr[] = {1,2,3};
+  test<long>();
+  test<const long>();
+  test<volatile long>();
+  test<const volatile long>();
 
-    static_assert(doCopy(std::span<      int>  ()),            "");
-    static_assert(doCopy(std::span<      int,0>()),            "");
-    static_assert(doCopy(std::span<const int>  (&carr[0], 1)), "");
-    static_assert(doCopy(std::span<const int,1>(&carr[0], 1)), "");
-    static_assert(doCopy(std::span<const int>  (&carr[0], 2)), "");
-    static_assert(doCopy(std::span<const int,2>(&carr[0], 2)), "");
+  test<double>();
+  test<const double>();
+  test<volatile double>();
+  test<const volatile double>();
 
-    static_assert(doCopy(std::span<long>()),   "");
-    static_assert(doCopy(std::span<double>()), "");
-    static_assert(doCopy(std::span<A>()),      "");
+  // Note: Can't test non-fundamental types with volatile because we require `T*` to be indirectly_readable,
+  //       which isn't the case when T is volatile.
+  test<Foo>();
+  test<const Foo>();
 
-    std::string s;
-    assert(doCopy(std::span<std::string>   ()     ));
-    assert(doCopy(std::span<std::string, 0>()     ));
-    assert(doCopy(std::span<std::string>   (&s, 1)));
-    assert(doCopy(std::span<std::string, 1>(&s, 1)));
+  test<std::string>();
+  test<const std::string>();
+
+  // Regression test for https://github.com/llvm/llvm-project/issues/104496
+  {
+    struct Incomplete;
+    std::span<Incomplete> x;
+    std::span<Incomplete> copy(x);
+    assert(copy.data() == x.data());
+    assert(copy.size() == x.size());
+  }
+
+  return true;
+}
 
-    testCV<               int>();
-    testCV<const          int>();
-    testCV<      volatile int>();
-    testCV<const volatile int>();
+int main(int, char**) {
+  test_all();
+  static_assert(test_all());
 
   return 0;
 }
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
index 99f55d3b17f1b09..3bebf38a2a7c34d 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
@@ -47,13 +47,20 @@ struct PathCompareTest {
   int expect;
 };
 
-#define LONGA "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
-#define LONGB "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
-#define LONGC "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC"
-#define LONGD "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"
-const PathCompareTest CompareTestCases[] =
-{
-    {"", "",  0},
+#define LONGA                                                                                                          \
+  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" \
+  "AAAAAAAA"
+#define LONGB                                                                                                          \
+  "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" \
+  "BBBBBBBB"
+#define LONGC                                                                                                          \
+  "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC" \
+  "CCCCCCCC"
+#define LONGD                                                                                                          \
+  "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" \
+  "DDDDDDDD"
+const PathCompareTest CompareTestCases[] = {
+    {"", "", 0},
     {"a", "", 1},
     {"", "a", -1},
     {"a/b/c", "a/b/c", 0},
@@ -62,14 +69,19 @@ const PathCompareTest CompareTestCases[] =
     {"a/b", "a/b/c", -1},
     {"a/b/c", "a/b", 1},
     {"a/b/", "a/b/.", -1},
-    {"a/b/", "a/b",    1},
+    {"a/b/", "a/b", 1},
     {"a/b//////", "a/b/////.", -1},
     {"a/.././b", "a///..//.////b", 0},
     {"//foo//bar///baz////", "//foo/bar/baz/", 0}, // duplicate separators
-    {"///foo/bar", "/foo/bar", 0}, // "///" is not a root directory
-    {"/foo/bar/", "/foo/bar", 1}, // trailing separator
+    {"///foo/bar", "/foo/bar", 0},                 // "///" is not a root directory
+    {"/foo/bar/", "/foo/bar", 1},                  // trailing separator
     {"foo", "/foo", -1}, // if !this->has_root_directory() and p.has_root_directory(), a value less than 0.
-    {"/foo", "foo", 1}, //  if this->has_root_directory() and !p.has_root_directory(), a value greater than 0.
+    {"/foo", "foo", 1},  //  if this->has_root_directory() and !p.has_root_directory(), a value greater than 0.
+#ifdef _WIN32
+    {"C:/a", "C:\\a", 0},
+#else
+    {"C:/a", "C:\\a", -1},
+#endif
     {("//" LONGA "////" LONGB "/" LONGC "///" LONGD), ("//" LONGA "/" LONGB "/" LONGC "/" LONGD), 0},
     {(LONGA "/" LONGB "/" LONGC), (LONGA "/" LONGB "/" LONGB), 1}
 
@@ -79,23 +91,19 @@ const PathCompareTest CompareTestCases[] =
 #undef LONGC
 #undef LONGD
 
-static inline int normalize_ret(int ret)
-{
-  return ret < 0 ? -1 : (ret > 0 ? 1 : 0);
-}
+static inline int normalize_ret(int ret) { return ret < 0 ? -1 : (ret > 0 ? 1 : 0); }
 
-void test_compare_basic()
-{
+void test_compare_basic() {
   using namespace fs;
-  for (auto const & TC : CompareTestCases) {
+  for (auto const& TC : CompareTestCases) {
     const path p1(TC.LHS);
     const path p2(TC.RHS);
     std::string RHS(TC.RHS);
     const path::string_type R(RHS.begin(), RHS.end());
     const std::basic_string_view<path::value_type> RV(R);
-    const path::value_type *Ptr = R.c_str();
-    const int E = TC.expect;
-    { // compare(...) functions
+    const path::value_type* Ptr = R.c_str();
+    const int E                 = TC.expect;
+    {                           // compare(...) functions
       DisableAllocationGuard g; // none of these operations should allocate
 
       // check runtime results
@@ -113,7 +121,7 @@ void test_compare_basic()
       // check signatures
       ASSERT_NOEXCEPT(p1.compare(p2));
     }
-    { // comparison operators
+    {                           // comparison operators
       DisableAllocationGuard g; // none of these operations should allocate
 
       // check signatures
@@ -180,14 +188,14 @@ void test_compare_elements() {
 
   auto BuildPath = [](std::vector<std::string> const& Elems) {
     fs::path p;
-    for (auto &E : Elems)
+    for (auto& E : Elems)
       p /= E;
     return p;
   };
 
-  for (auto &TC : TestCases) {
-    fs::path LHS = BuildPath(TC.LHSElements);
-    fs::path RHS = BuildPath(TC.RHSElements);
+  for (auto& TC : TestCases) {
+    fs::path LHS        = BuildPath(TC.LHSElements);
+    fs::path RHS        = BuildPath(TC.RHSElements);
     const int ExpectCmp = CompareElements(TC.LHSElements, TC.RHSElements);
     assert(ExpectCmp == TC.Expect);
     const int GotCmp = normalize_ret(LHS.compare(RHS));
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp
index 42fa813ada4a5c8..06954fe876e5572 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp
@@ -41,9 +41,11 @@ int main(int, char**) {
 #ifdef _WIN32
       {"//net/", "//net", ""},
       {"//net", "//net/", ""},
+      {"C:\\a\\b", "C:/a", "b"},
 #else
       {"//net/", "//net", "."},
       {"//net", "//net/", "."},
+      {"C:\\a\\b", "C:/a", "../../C:\\a\\b"},
 #endif
       {"//base", "a", ""},
       {"a", "a", "."},
diff --git a/libcxx/test/std/numerics/c.math/cmath.pass.cpp b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
index 6028aa5a2110342..34c30fb998f4746 100644
--- a/libcxx/test/std/numerics/c.math/cmath.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
@@ -708,15 +708,16 @@ void test_isinf()
     static_assert((std::is_same<decltype(std::isinf((float)0)), bool>::value), "");
 
     typedef decltype(std::isinf((double)0)) DoubleRetType;
-#if !defined(__linux__) || defined(__clang__)
-    static_assert((std::is_same<DoubleRetType, bool>::value), "");
-#else
+#if defined(__GLIBC__) && TEST_STD_VER == 03 && defined(TEST_COMPILER_CLANG)
     // GLIBC < 2.23 defines 'isinf(double)' with a return type of 'int' in
     // all C++ dialects. The test should tolerate this when libc++ can't work
-    // around it.
+    // around it via `_LIBCPP_PREFERRED_OVERLOAD`, which is only available
+    // in modern versions of Clang, and not elsewhere.
     // See: https://sourceware.org/bugzilla/show_bug.cgi?id=19439
     static_assert((std::is_same<DoubleRetType, bool>::value
                 || std::is_same<DoubleRetType, int>::value), "");
+#else
+    static_assert((std::is_same<DoubleRetType, bool>::value), "");
 #endif
 
     static_assert((std::is_same<decltype(std::isinf(0)), bool>::value), "");
@@ -794,15 +795,16 @@ void test_isnan()
     static_assert((std::is_same<decltype(std::isnan((float)0)), bool>::value), "");
 
     typedef decltype(std::isnan((double)0)) DoubleRetType;
-#if !defined(__linux__) || defined(__clang__)
-    static_assert((std::is_same<DoubleRetType, bool>::value), "");
-#else
-    // GLIBC < 2.23 defines 'isinf(double)' with a return type of 'int' in
+#if defined(__GLIBC__) && TEST_STD_VER == 03 && defined(TEST_COMPILER_CLANG)
+    // GLIBC < 2.23 defines 'isnan(double)' with a return type of 'int' in
     // all C++ dialects. The test should tolerate this when libc++ can't work
-    // around it.
+    // around it via `_LIBCPP_PREFERRED_OVERLOAD`, which is only available
+    // in modern versions of Clang, and not elsewhere.
     // See: https://sourceware.org/bugzilla/show_bug.cgi?id=19439
     static_assert((std::is_same<DoubleRetType, bool>::value
                 || std::is_same<DoubleRetType, int>::value), "");
+#else
+    static_assert((std::is_same<DoubleRetType, bool>::value), "");
 #endif
 
     static_assert((std::is_same<decltype(std::isnan(0)), bool>::value), "");
diff --git a/libcxx/test/std/numerics/c.math/isinf.pass.cpp b/libcxx/test/std/numerics/c.math/isinf.pass.cpp
index e935b53187fe6ce..e5169e8056c2ea1 100644
--- a/libcxx/test/std/numerics/c.math/isinf.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/isinf.pass.cpp
@@ -62,9 +62,21 @@ struct TestInt {
   }
 };
 
+template <typename T>
+struct ConvertibleTo {
+  operator T() const { return T(); }
+};
+
 int main(int, char**) {
   types::for_each(types::floating_point_types(), TestFloat());
   types::for_each(types::integral_types(), TestInt());
 
+  // Make sure we can call `std::isinf` with convertible types
+  {
+    assert(!std::isinf(ConvertibleTo<float>()));
+    assert(!std::isinf(ConvertibleTo<double>()));
+    assert(!std::isinf(ConvertibleTo<long double>()));
+  }
+
   return 0;
 }
diff --git a/libcxx/test/std/numerics/c.math/isnan.pass.cpp b/libcxx/test/std/numerics/c.math/isnan.pass.cpp
index fffb12464586292..e4ccab1243e5678 100644
--- a/libcxx/test/std/numerics/c.math/isnan.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/isnan.pass.cpp
@@ -62,9 +62,21 @@ struct TestInt {
   }
 };
 
+template <typename T>
+struct ConvertibleTo {
+  operator T() const { return T(); }
+};
+
 int main(int, char**) {
   types::for_each(types::floating_point_types(), TestFloat());
   types::for_each(types::integral_types(), TestInt());
 
+  // Make sure we can call `std::isnan` with convertible types
+  {
+    assert(!std::isnan(ConvertibleTo<float>()));
+    assert(!std::isnan(ConvertibleTo<double>()));
+    assert(!std::isnan(ConvertibleTo<long double>()));
+  }
+
   return 0;
 }
diff --git a/libcxx/test/support/count_new.h b/libcxx/test/support/count_new.h
index 2298c4fd63e845b..61c8ca16ab0d043 100644
--- a/libcxx/test/support/count_new.h
+++ b/libcxx/test/support/count_new.h
@@ -455,11 +455,23 @@ void operator delete[](void* p, std::nothrow_t const&) TEST_NOEXCEPT {
 #      define USE_ALIGNED_ALLOC
 #    endif
 
+#    if defined(__APPLE__)
+#      if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) &&                                                   \
+           __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500)
+#        define TEST_HAS_NO_C11_ALIGNED_ALLOC
+#      endif
+#    elif defined(__ANDROID__) && __ANDROID_API__ < 28
+#      define TEST_HAS_NO_C11_ALIGNED_ALLOC
+#    endif
+
 inline void* allocate_aligned_impl(std::size_t size, std::align_val_t align) {
   const std::size_t alignment = static_cast<std::size_t>(align);
   void* ret                   = nullptr;
 #    ifdef USE_ALIGNED_ALLOC
   ret = _aligned_malloc(size, alignment);
+#    elif TEST_STD_VER >= 17 && !defined(TEST_HAS_NO_C11_ALIGNED_ALLOC)
+  size_t rounded_size = (size + alignment - 1) & ~(alignment - 1);
+  ret                 = aligned_alloc(alignment, size > rounded_size ? size : rounded_size);
 #    else
   assert(posix_memalign(&ret, std::max(alignment, sizeof(void*)), size) != EINVAL);
 #    endif
diff --git a/libcxx/test/support/platform_support.h b/libcxx/test/support/platform_support.h
index 0d4fa63b03f107c..99e60f60c5998f6 100644
--- a/libcxx/test/support/platform_support.h
+++ b/libcxx/test/support/platform_support.h
@@ -78,7 +78,7 @@ inline std::string get_temp_file_name() {
   for (std::size_t i = 0; i < sizeof(Name); ++i)
     if (Name[i] == 'X')
       Name[i] = chars[rand() % strlen(chars)];
-  FILE* file = fopen(filename, "w");
+  FILE* file = fopen(Name, "w");
   if (!file)
     abort();
   if (fclose(file) == EOF)
diff --git a/libcxxabi/src/cxa_default_handlers.cpp b/libcxxabi/src/cxa_default_handlers.cpp
index 7bcfca069fb7f71..60e402c55b395b6 100644
--- a/libcxxabi/src/cxa_default_handlers.cpp
+++ b/libcxxabi/src/cxa_default_handlers.cpp
@@ -10,8 +10,6 @@
 //===----------------------------------------------------------------------===//
 
 #include <exception>
-#include <memory>
-#include <stdlib.h>
 #include "abort_message.h"
 #include "cxxabi.h"
 #include "cxa_handlers.h"
@@ -23,17 +21,7 @@
 
 static constinit const char* cause = "uncaught";
 
-#ifndef _LIBCXXABI_NO_EXCEPTIONS
-// Demangle the given string, or return the string as-is in case of an error.
-static std::unique_ptr<char const, void (*)(char const*)> demangle(char const* str)
-{
-#if !defined(LIBCXXABI_NON_DEMANGLING_TERMINATE)
-    if (const char* result = __cxxabiv1::__cxa_demangle(str, nullptr, nullptr, nullptr))
-        return {result, [](char const* p) { std::free(const_cast<char*>(p)); }};
-#endif
-    return {str, [](char const*) { /* nothing to free */ }};
-}
-
+#  ifndef _LIBCXXABI_NO_EXCEPTIONS
 __attribute__((noreturn))
 static void demangling_terminate_handler()
 {
@@ -61,7 +49,17 @@ static void demangling_terminate_handler()
             exception_header + 1;
     const __shim_type_info* thrown_type =
         static_cast<const __shim_type_info*>(exception_header->exceptionType);
-    auto name = demangle(thrown_type->name());
+
+    auto name = [str = thrown_type->name()] {
+#    ifndef LIBCXXABI_NON_DEMANGLING_TERMINATE
+      if (const char* result = __cxxabiv1::__cxa_demangle(str, nullptr, nullptr, nullptr))
+        // We're about to abort(), this memory can never be freed; so it's fine
+        // to just return a raw pointer
+        return result;
+#    endif
+      return str;
+    }();
+
     // If the uncaught exception can be caught with std::exception&
     const __shim_type_info* catch_type =
         static_cast<const __shim_type_info*>(&typeid(std::exception));
@@ -69,12 +67,12 @@ static void demangling_terminate_handler()
     {
         // Include the what() message from the exception
         const std::exception* e = static_cast<const std::exception*>(thrown_object);
-        abort_message("terminating due to %s exception of type %s: %s", cause, name.get(), e->what());
+        abort_message("terminating due to %s exception of type %s: %s", cause, name, e->what());
     }
     else
     {
         // Else just note that we're terminating due to an exception
-        abort_message("terminating due to %s exception of type %s", cause, name.get());
+        abort_message("terminating due to %s exception of type %s", cause, name);
     }
 }
 #else // !_LIBCXXABI_NO_EXCEPTIONS
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index 0c4ba1abb4778c2..01c15d79e748aed 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -106,10 +106,13 @@ static StringRef getOutputSectionName(const InputSectionBase *s) {
     return ".text";
   }
 
-  for (StringRef v :
-       {".data.rel.ro", ".data", ".rodata", ".bss.rel.ro", ".bss", ".ldata",
-        ".lrodata", ".lbss", ".gcc_except_table", ".init_array", ".fini_array",
-        ".tbss", ".tdata", ".ARM.exidx", ".ARM.extab", ".ctors", ".dtors"})
+  for (StringRef v : {".data.rel.ro", ".data",       ".rodata",
+                      ".bss.rel.ro",  ".bss",        ".ldata",
+                      ".lrodata",     ".lbss",       ".gcc_except_table",
+                      ".init_array",  ".fini_array", ".tbss",
+                      ".tdata",       ".ARM.exidx",  ".ARM.extab",
+                      ".ctors",       ".dtors",      ".sbss",
+                      ".sdata",       ".srodata"})
     if (isSectionPrefix(v, s->name))
       return v;
 
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index f3d2a93914f7176..6a1ff96ed656975 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -1042,20 +1042,36 @@ static bool shouldAdhocSignByDefault(Architecture arch, PlatformType platform) {
          platform == PLATFORM_XROS_SIMULATOR;
 }
 
-static bool dataConstDefault(const InputArgList &args) {
-  static const std::array<std::pair<PlatformType, VersionTuple>, 6> minVersion =
-      {{{PLATFORM_MACOS, VersionTuple(10, 15)},
-        {PLATFORM_IOS, VersionTuple(13, 0)},
-        {PLATFORM_TVOS, VersionTuple(13, 0)},
-        {PLATFORM_WATCHOS, VersionTuple(6, 0)},
-        {PLATFORM_XROS, VersionTuple(1, 0)},
-        {PLATFORM_BRIDGEOS, VersionTuple(4, 0)}}};
-  PlatformType platform = removeSimulator(config->platformInfo.target.Platform);
-  auto it = llvm::find_if(minVersion,
+template <std::size_t N>
+using MinVersions = std::array<std::pair<PlatformType, VersionTuple>, N>;
+
+/// Returns true if the platform is greater than the min version.
+/// Returns false if the platform does not exist.
+template <std::size_t N>
+static bool greaterEqMinVersion(const MinVersions<N> &minVersions,
+                                bool ignoreSimulator) {
+  PlatformType platform = config->platformInfo.target.Platform;
+  if (ignoreSimulator)
+    platform = removeSimulator(platform);
+  auto it = llvm::find_if(minVersions,
                           [&](const auto &p) { return p.first == platform; });
-  if (it != minVersion.end())
-    if (config->platformInfo.target.MinDeployment < it->second)
-      return false;
+  if (it != minVersions.end())
+    if (config->platformInfo.target.MinDeployment >= it->second)
+      return true;
+  return false;
+}
+
+static bool dataConstDefault(const InputArgList &args) {
+  static const MinVersions<6> minVersion = {{
+      {PLATFORM_MACOS, VersionTuple(10, 15)},
+      {PLATFORM_IOS, VersionTuple(13, 0)},
+      {PLATFORM_TVOS, VersionTuple(13, 0)},
+      {PLATFORM_WATCHOS, VersionTuple(6, 0)},
+      {PLATFORM_XROS, VersionTuple(1, 0)},
+      {PLATFORM_BRIDGEOS, VersionTuple(4, 0)},
+  }};
+  if (!greaterEqMinVersion(minVersion, true))
+    return false;
 
   switch (config->outputType) {
   case MH_EXECUTE:
@@ -1106,30 +1122,18 @@ static bool shouldEmitChainedFixups(const InputArgList &args) {
   if (requested)
     return true;
 
-  static const std::array<std::pair<PlatformType, VersionTuple>, 9> minVersion =
-      {{
-          {PLATFORM_IOS, VersionTuple(13, 4)},
-          {PLATFORM_IOSSIMULATOR, VersionTuple(16, 0)},
-          {PLATFORM_MACOS, VersionTuple(13, 0)},
-          {PLATFORM_TVOS, VersionTuple(14, 0)},
-          {PLATFORM_TVOSSIMULATOR, VersionTuple(15, 0)},
-          {PLATFORM_WATCHOS, VersionTuple(7, 0)},
-          {PLATFORM_WATCHOSSIMULATOR, VersionTuple(8, 0)},
-          {PLATFORM_XROS, VersionTuple(1, 0)},
-          {PLATFORM_XROS_SIMULATOR, VersionTuple(1, 0)},
-      }};
-  PlatformType platform = config->platformInfo.target.Platform;
-  auto it = llvm::find_if(minVersion,
-                          [&](const auto &p) { return p.first == platform; });
-
-  // We don't know the versions for other platforms, so default to disabled.
-  if (it == minVersion.end())
-    return false;
-
-  if (it->second > config->platformInfo.target.MinDeployment)
-    return false;
-
-  return true;
+  static const MinVersions<9> minVersion = {{
+      {PLATFORM_IOS, VersionTuple(13, 4)},
+      {PLATFORM_IOSSIMULATOR, VersionTuple(16, 0)},
+      {PLATFORM_MACOS, VersionTuple(13, 0)},
+      {PLATFORM_TVOS, VersionTuple(14, 0)},
+      {PLATFORM_TVOSSIMULATOR, VersionTuple(15, 0)},
+      {PLATFORM_WATCHOS, VersionTuple(7, 0)},
+      {PLATFORM_WATCHOSSIMULATOR, VersionTuple(8, 0)},
+      {PLATFORM_XROS, VersionTuple(1, 0)},
+      {PLATFORM_XROS_SIMULATOR, VersionTuple(1, 0)},
+  }};
+  return greaterEqMinVersion(minVersion, false);
 }
 
 static bool shouldEmitRelativeMethodLists(const InputArgList &args) {
@@ -1140,12 +1144,20 @@ static bool shouldEmitRelativeMethodLists(const InputArgList &args) {
   if (arg && arg->getOption().getID() == OPT_no_objc_relative_method_lists)
     return false;
 
-  // TODO: If no flag is specified, don't default to false, but instead:
-  //   - default false on   <   ios14
-  //   - default true  on   >=  ios14
-  // For now, until this feature is confirmed stable, default to false if no
-  // flag is explicitly specified
-  return false;
+  // If no flag is specified, enable this on newer versions by default.
+  // The min versions is taken from
+  // ld64(https://github.com/apple-oss-distributions/ld64/blob/47f477cb721755419018f7530038b272e9d0cdea/src/ld/ld.hpp#L310)
+  // to mimic to operation of ld64
+  // [here](https://github.com/apple-oss-distributions/ld64/blob/47f477cb721755419018f7530038b272e9d0cdea/src/ld/Options.cpp#L6085-L6101)
+  static const MinVersions<6> minVersion = {{
+      {PLATFORM_MACOS, VersionTuple(10, 16)},
+      {PLATFORM_IOS, VersionTuple(14, 0)},
+      {PLATFORM_WATCHOS, VersionTuple(7, 0)},
+      {PLATFORM_TVOS, VersionTuple(14, 0)},
+      {PLATFORM_BRIDGEOS, VersionTuple(5, 0)},
+      {PLATFORM_XROS, VersionTuple(1, 0)},
+  }};
+  return greaterEqMinVersion(minVersion, true);
 }
 
 void SymbolPatterns::clear() {
diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index 6b4ec4989ca4a11..939e9b286d77f59 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -2119,7 +2119,7 @@ void ObjCMethListSection::writeRelativeOffsetForIsec(
     assert(selRef && "Expected all selector names to already be already be "
                      "present in __objc_selrefs");
     symVA = selRef->getVA();
-    assert(selRef->data.size() == sizeof(target->wordSize) &&
+    assert(selRef->data.size() == target->wordSize &&
            "Expected one selref per ConcatInputSection");
   } else if (reloc->referent.is<Symbol *>()) {
     auto *def = dyn_cast_or_null<Defined>(reloc->referent.get<Symbol *>());
diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp
index 35fd478a21905e7..c7d7b9cfca386f5 100644
--- a/lld/MinGW/Driver.cpp
+++ b/lld/MinGW/Driver.cpp
@@ -448,6 +448,9 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
       add("-errorlimit:" + s);
   }
 
+  if (auto *a = args.getLastArg(OPT_rpath))
+    warn("parameter " + a->getSpelling() + " has no effect on PE/COFF targets");
+
   for (auto *a : args.filtered(OPT_mllvm))
     add("-mllvm:" + StringRef(a->getValue()));
 
diff --git a/lld/MinGW/Options.td b/lld/MinGW/Options.td
index 56f67e3dd96c426..7bd5fb80749da2c 100644
--- a/lld/MinGW/Options.td
+++ b/lld/MinGW/Options.td
@@ -243,6 +243,9 @@ defm: EqNoHelp<"sysroot">;
 def: F<"sort-common">;
 def: F<"start-group">;
 
+// Ignored options, that produce warnings
+defm rpath: EqNoHelp<"rpath">;
+
 // Ignore GCC collect2 LTO plugin related options. Note that we don't support
 // GCC LTO, but GCC collect2 passes these options even in non-LTO mode.
 def: J<"plugin-opt=-fresolution=">;
diff --git a/lld/test/ELF/riscv-section-layout.s b/lld/test/ELF/riscv-section-layout.s
index 10e0febf7c4260a..123b5f056ad66ba 100644
--- a/lld/test/ELF/riscv-section-layout.s
+++ b/lld/test/ELF/riscv-section-layout.s
@@ -15,12 +15,13 @@
 # RUN: ld.lld -pie %t.64s.o -o %t.64s
 # RUN: llvm-readelf -S -sX %t.64s | FileCheck %s
 
-# NOSDATA:      .text
+# NOSDATA:      .rodata  PROGBITS
+# NOSDATA-NEXT: .text    PROGBITS
 # NOSDATA-NEXT: .tdata   PROGBITS [[#%x,TDATA:]]
-# NOSDATA-NEXT: .tbss
-# NOSDATA-NEXT: .dynamic
-# NOSDATA-NEXT: .got
-# NOSDATA-NEXT: .relro_padding
+# NOSDATA-NEXT: .tbss    NOBITS
+# NOSDATA-NEXT: .dynamic DYNAMIC
+# NOSDATA-NEXT: .got     PROGBITS
+# NOSDATA-NEXT: .relro_padding NOBITS
 # NOSDATA-NEXT: .data    PROGBITS [[#%x,DATA:]]
 # NOSDATA-NEXT: .bss     NOBITS   [[#%x,BSS:]]
 
@@ -33,16 +34,18 @@
 # NOSDATA-DAG:  [[#]]: {{0*}}[[#BSS]]         0 NOTYPE  GLOBAL DEFAULT [[#]] (.bss) __bss_start
 # NOSDATA-DAG:  [[#]]: {{0*}}800              0 NOTYPE  GLOBAL DEFAULT  1 (.dynsym) __global_pointer$
 
-# CHECK:      .text
-# CHECK-NEXT: .tdata
-# CHECK-NEXT: .tbss
-# CHECK-NEXT: .dynamic
-# CHECK-NEXT: .got
-# CHECK-NEXT: .relro_padding
-# CHECK-NEXT: .data
+# CHECK:      .rodata    PROGBITS
+# CHECK-NEXT: .srodata   PROGBITS
+# CHECK-NEXT: .text      PROGBITS
+# CHECK-NEXT: .tdata     PROGBITS
+# CHECK-NEXT: .tbss      NOBITS
+# CHECK-NEXT: .dynamic   DYNAMIC
+# CHECK-NEXT: .got       PROGBITS
+# CHECK-NEXT: .relro_padding NOBITS
+# CHECK-NEXT: .data      PROGBITS
 # CHECK-NEXT: .sdata     PROGBITS [[#%x,SDATA:]]
 # CHECK-NEXT: .sbss      NOBITS   [[#%x,SBSS:]]
-# CHECK-NEXT: .bss
+# CHECK-NEXT: .bss       NOBITS
 
 # CHECK-DAG:  [[#]]: {{0*}}[[#SBSS]]        0 NOTYPE  GLOBAL DEFAULT [[#]] (.sdata) _edata
 # CHECK-DAG:  [[#]]: {{0*}}[[#SBSS]]        0 NOTYPE  GLOBAL DEFAULT [[#]] (.sbss) __bss_start
@@ -51,11 +54,16 @@
 .globl _etext, _edata, __bss_start
   lla gp, __global_pointer$
 
+.section .rodata,"a",@progbits; .space 1
 .section .data,"aw",@progbits; .long _GLOBAL_OFFSET_TABLE_ - .
 .section .bss,"aw",@nobits; .space 1
 .section .tdata,"awT",@progbits; .space 1
 .section .tbss,"awT",@nobits; .space 1
 .ifdef SDATA
 .section .sdata,"aw",@progbits; .space 1
+.section .sdata.suffix,"aw",@progbits; .space 1
 .section .sbss,"aw",@nobits; .space 1
+.section .sbss.suffix,"aw",@nobits; .space 1
+.section .srodata,"a",@progbits; .space 1
+.section .srodata.suffix,"a",@progbits; .space 1
 .endif
diff --git a/lld/test/ELF/section-name.s b/lld/test/ELF/section-name.s
index 819cd9c14d95001..7793cc547295996 100644
--- a/lld/test/ELF/section-name.s
+++ b/lld/test/ELF/section-name.s
@@ -41,19 +41,34 @@ _start:
 .byte 0
 .section .tdata.foo,"aGwT",@progbits,foo,comdat
 .byte 0
+.section .sdata,"aw"
+.byte 0
+.section .sdata.foo,"aw"
+.byte 0
+.section .sbss,"aw",@nobits
+.byte 0
+.section .sbss.foo,"aw",@nobits
+.byte 0
+.section .srodata,"a"
+.byte 0
+.section .srodata.foo,"a"
+.byte 0
 
 // CHECK:      .rodata           00000002
 // CHECK-NEXT: .gcc_except_table 00000001
+// CHECK-NEXT: .srodata          00000002
 // CHECK-NEXT: .text             00000002
 // CHECK-NEXT: .tdata            00000001
 // CHECK-NEXT: .tbss             00000001
 // CHECK-NEXT: .data.rel.ro      00000004
-// CHECK-NEXT: .relro_padding    00000df5
+// CHECK-NEXT: .relro_padding    00000df1
 // CHECK-NEXT: .data             00000002
 // CHECK-NEXT: .foo.a            00000001
 // CHECK-NEXT: .foo              00000001
+// CHECK-NEXT: .sdata            00000002
 // CHECK-NEXT: .bss              00000002
+// CHECK-NEXT: .sbss             00000002
 // CHECK-NEXT: .comment          00000008
 // CHECK-NEXT: .symtab           00000030
-// CHECK-NEXT: .shstrtab         00000084
+// CHECK-NEXT: .shstrtab         0000009a
 // CHECK-NEXT: .strtab           00000008
diff --git a/lld/test/MachO/objc-category-conflicts.s b/lld/test/MachO/objc-category-conflicts.s
index eb4c0adfd9e934b..3fb81d7caf092c6 100644
--- a/lld/test/MachO/objc-category-conflicts.s
+++ b/lld/test/MachO/objc-category-conflicts.s
@@ -10,31 +10,31 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos11.0 -I %t %t/cat2.s --defsym MAKE_LOAD_METHOD=1 -o %t/cat2-with-load.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos11.0 -I %t %t/klass.s --defsym MAKE_LOAD_METHOD=1 -o %t/klass-with-load.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos11.0 -I %t %t/klass-with-no-rodata.s -o %t/klass-with-no-rodata.o
-# RUN: %lld -dylib -lobjc %t/klass.o -o %t/libklass.dylib
+# RUN: %lld -no_objc_relative_method_lists -dylib -lobjc %t/klass.o -o %t/libklass.dylib
 
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/klass.o %t/cat1.o %t/cat2.o -o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/klass.o %t/cat1.o %t/cat2.o -o \
 # RUN:   /dev/null 2>&1 | FileCheck %s --check-prefixes=CATCLS,CATCAT
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/libklass.dylib %t/cat1.o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/libklass.dylib %t/cat1.o \
 # RUN:   %t/cat2.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=CATCAT
 
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/klass_w_sym.o %t/cat1_w_sym.o %t/cat2_w_sym.o -o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/klass_w_sym.o %t/cat1_w_sym.o %t/cat2_w_sym.o -o \
 # RUN:   /dev/null 2>&1 | FileCheck %s --check-prefixes=CATCLS_W_SYM,CATCAT_W_SYM
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/libklass.dylib %t/cat1_w_sym.o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/libklass.dylib %t/cat1_w_sym.o \
 # RUN:   %t/cat2_w_sym.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=CATCAT_W_SYM
 
 ## Check that we don't emit spurious warnings around the +load method while
 ## still emitting the other warnings. Note that we have made separate
 ## `*-with-load.s` files for ease of comparison with ld64; ld64 will not warn
 ## at all if multiple +load methods are present.
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/klass-with-load.o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/klass-with-load.o \
 # RUN:   %t/cat1-with-load.o %t/cat2-with-load.o -o /dev/null 2>&1 | \
 # RUN:   FileCheck %s --check-prefixes=CATCLS,CATCAT --implicit-check-not '+load'
 
 ## Regression test: Check that we don't crash.
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/klass-with-no-rodata.o -o /dev/null
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/klass-with-no-rodata.o -o /dev/null
 
 ## Check that we don't emit any warnings without --check-category-conflicts.
-# RUN: %no-fatal-warnings-lld -dylib -lobjc %t/klass.o %t/cat1.o %t/cat2.o -o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists -dylib -lobjc %t/klass.o %t/cat1.o %t/cat2.o -o \
 # RUN:   /dev/null 2>&1 | FileCheck %s --implicit-check-not 'warning' --allow-empty
 
 # CATCLS:      warning: method '+s1' has conflicting definitions:
diff --git a/lld/test/MachO/objc-category-merging-complete-test.s b/lld/test/MachO/objc-category-merging-complete-test.s
index cb112073eb87103..3befd683e3a15ef 100644
--- a/lld/test/MachO/objc-category-merging-complete-test.s
+++ b/lld/test/MachO/objc-category-merging-complete-test.s
@@ -7,10 +7,10 @@
 # RUN: %lld -arch arm64 a64_file1.o -o a64_file1.dylib -dylib
 
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o a64_file2.o a64_file2.s
-# RUN: %lld -arch arm64 -o a64_file2_no_merge.exe a64_file1.dylib a64_file2.o
-# RUN: %lld -arch arm64 -o a64_file2_no_merge_v2.exe a64_file1.dylib a64_file2.o -no_objc_category_merging
-# RUN: %lld -arch arm64 -o a64_file2_no_merge_v3.exe a64_file1.dylib a64_file2.o -objc_category_merging -no_objc_category_merging
-# RUN: %lld -arch arm64 -o a64_file2_merge.exe -objc_category_merging a64_file1.dylib a64_file2.o
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_no_merge.exe a64_file1.dylib a64_file2.o
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_no_merge_v2.exe a64_file1.dylib a64_file2.o -no_objc_category_merging
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_no_merge_v3.exe a64_file1.dylib a64_file2.o -objc_category_merging -no_objc_category_merging
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_merge.exe -objc_category_merging a64_file1.dylib a64_file2.o
 
 # RUN: llvm-objdump --objc-meta-data --macho a64_file2_no_merge.exe | FileCheck %s --check-prefixes=NO_MERGE_CATS
 # RUN: llvm-objdump --objc-meta-data --macho a64_file2_no_merge_v2.exe | FileCheck %s --check-prefixes=NO_MERGE_CATS
@@ -18,7 +18,7 @@
 # RUN: llvm-objdump --objc-meta-data --macho a64_file2_merge.exe | FileCheck %s --check-prefixes=MERGE_CATS
 
 ############ Test merging multiple categories into the base class ############
-# RUN: %lld -arch arm64 -o a64_file2_merge_into_class.exe -objc_category_merging a64_file1.o a64_file2.o
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_merge_into_class.exe -objc_category_merging a64_file1.o a64_file2.o
 # RUN: llvm-objdump --objc-meta-data --macho a64_file2_merge_into_class.exe | FileCheck %s --check-prefixes=MERGE_CATS_CLS
 
 
diff --git a/lld/test/MachO/objc-category-merging-erase-objc-name-test.s b/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
index aeb2395b3a85805..9e4046a1447e703 100644
--- a/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
+++ b/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
@@ -5,7 +5,7 @@
 ; delete the 'MyTestProtocol' name
 
 ; RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o %T/erase-objc-name.o %s
-; RUN: %lld -arch arm64 -dylib -o %T/erase-objc-name.dylib %T/erase-objc-name.o -objc_category_merging
+; RUN: %lld -no_objc_relative_method_lists -arch arm64 -dylib -o %T/erase-objc-name.dylib %T/erase-objc-name.o -objc_category_merging
 ; RUN: llvm-objdump --objc-meta-data --macho %T/erase-objc-name.dylib | FileCheck %s --check-prefixes=MERGE_CATS
 
 ; === Check merge categories enabled ===
diff --git a/lld/test/MachO/objc-category-merging-minimal.s b/lld/test/MachO/objc-category-merging-minimal.s
index b94799a57a4d852..088a4d0f30417f6 100644
--- a/lld/test/MachO/objc-category-merging-minimal.s
+++ b/lld/test/MachO/objc-category-merging-minimal.s
@@ -9,7 +9,7 @@
 ## Create our main testing dylib - linking against the fake dylib above
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o merge_cat_minimal.o merge_cat_minimal.s
 # RUN: %lld -arch arm64 -dylib -o merge_cat_minimal_no_merge.dylib a64_fakedylib.dylib merge_cat_minimal.o
-# RUN: %lld -objc_relative_method_lists -arch arm64 -dylib -o merge_cat_minimal_merge.dylib -objc_category_merging a64_fakedylib.dylib merge_cat_minimal.o
+# RUN: %lld -arch arm64 -dylib -o merge_cat_minimal_merge.dylib -objc_category_merging a64_fakedylib.dylib merge_cat_minimal.o
 
 ## Now verify that the flag caused category merging to happen appropriatelly
 # RUN: llvm-objdump --objc-meta-data --macho merge_cat_minimal_no_merge.dylib | FileCheck %s --check-prefixes=NO_MERGE_CATS
@@ -17,7 +17,7 @@
 
 ############ Test merging multiple categories into the base class ############
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o merge_base_class_minimal.o merge_base_class_minimal.s
-# RUN: %lld -arch arm64 -dylib -objc_relative_method_lists -o merge_base_class_minimal_yes_merge.dylib -objc_category_merging merge_base_class_minimal.o merge_cat_minimal.o
+# RUN: %lld -arch arm64 -dylib -o merge_base_class_minimal_yes_merge.dylib -objc_category_merging merge_base_class_minimal.o merge_cat_minimal.o
 # RUN: %lld -arch arm64 -dylib -o merge_base_class_minimal_no_merge.dylib merge_base_class_minimal.o merge_cat_minimal.o
 
 # RUN: llvm-objdump --objc-meta-data --macho merge_base_class_minimal_no_merge.dylib  | FileCheck %s --check-prefixes=NO_MERGE_INTO_BASE
@@ -25,7 +25,7 @@
 
 ############ Test merging swift category into the base class ############
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o MyBaseClassSwiftExtension.o MyBaseClassSwiftExtension.s
-# RUN: %lld -arch arm64 -dylib -o merge_base_class_swift_minimal_yes_merge.dylib -objc_category_merging MyBaseClassSwiftExtension.o merge_base_class_minimal.o
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -dylib -o merge_base_class_swift_minimal_yes_merge.dylib -objc_category_merging MyBaseClassSwiftExtension.o merge_base_class_minimal.o
 # RUN: llvm-objdump --objc-meta-data --macho merge_base_class_swift_minimal_yes_merge.dylib | FileCheck %s --check-prefixes=YES_MERGE_INTO_BASE_SWIFT
 
 #### Check merge categories enabled ###
diff --git a/lld/test/MachO/objc-category-merging-swift.s b/lld/test/MachO/objc-category-merging-swift.s
index 28025b0bc577a88..7a084d9310d6c62 100644
--- a/lld/test/MachO/objc-category-merging-swift.s
+++ b/lld/test/MachO/objc-category-merging-swift.s
@@ -5,7 +5,7 @@
 ## Apply category merging to swiftc code just make sure we can handle addends
 ## and don't erase category names for swift -- in order to not crash
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o cat_swift.o %s
-# RUN: %lld -arch arm64 -dylib -o cat_swift.dylib cat_swift.o -objc_category_merging
+# RUN: %lld -arch arm64 -dylib -o cat_swift.dylib cat_swift.o -objc_category_merging -no_objc_relative_method_lists
 # RUN: llvm-objdump --objc-meta-data --macho cat_swift.dylib | FileCheck %s --check-prefixes=CHECK-MERGE
 
 ; CHECK-MERGE:      Contents of (__DATA_CONST,__objc_classlist) section
diff --git a/lld/test/MachO/objc-relative-method-lists-simple.s b/lld/test/MachO/objc-relative-method-lists-simple.s
index 9f54b5ad828a076..c8646f59603b3f3 100644
--- a/lld/test/MachO/objc-relative-method-lists-simple.s
+++ b/lld/test/MachO/objc-relative-method-lists-simple.s
@@ -3,20 +3,28 @@
 # RUN: rm -rf %t; split-file %s %t && cd %t
 
 ## Compile a64_rel_dylib.o
-# RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o a64_rel_dylib.o a64_simple_class.s
+# RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos10.15 -o a64_rel_dylib.o a64_simple_class.s
 
 ## Test arm64 + relative method lists
-# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -objc_relative_method_lists
+# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64
 # RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_REL
 
 ## Test arm64 + relative method lists + dead-strip
-# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -objc_relative_method_lists -dead_strip
+# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -dead_strip
 # RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_REL
 
 ## Test arm64 + traditional method lists (no relative offsets)
 # RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -no_objc_relative_method_lists
 # RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_NO_REL
 
+## Test arm64 + relative method lists by explicitly adding `-objc_relative_method_lists`.
+# RUN: %lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -platform_version macOS 10.15 10.15 -objc_relative_method_lists
+# RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_REL
+
+## Test arm64 + no relative method lists by default.
+# RUN: %lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -platform_version macOS 10.15 10.15
+# RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_NO_REL
+
 
 CHK_REL:       Contents of (__DATA_CONST,__objc_classlist) section
 CHK_REL-NEXT:  _OBJC_CLASS_$_MyClass
@@ -125,7 +133,7 @@ CHK_NO_REL-NEXT:		       imp +[MyClass class_method_02]
 .include "objc-macros.s"
 
 .section	__TEXT,__text,regular,pure_instructions
-.build_version macos, 11, 0
+.build_version macos, 10, 15
 
 .objc_selector_def "-[MyClass instance_method_00]"
 .objc_selector_def "-[MyClass instance_method_01]"
diff --git a/lld/test/MinGW/driver.test b/lld/test/MinGW/driver.test
index cbccd09793c2f0d..0dab66b613c774f 100644
--- a/lld/test/MinGW/driver.test
+++ b/lld/test/MinGW/driver.test
@@ -446,3 +446,9 @@ RUN: ld.lld -### foo.o -m i386pep --build-id=none 2>&1 | FileCheck -check-prefix
 RUN: ld.lld -### foo.o -m i386pep -s 2>&1 | FileCheck -check-prefix=NO_BUILD_ID %s
 RUN: ld.lld -### foo.o -m i386pep -S 2>&1 | FileCheck -check-prefix=NO_BUILD_ID %s
 NO_BUILD_ID: -build-id:no
+
+RUN: ld.lld -### foo.o -m i386pep -rpath foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
+RUN: ld.lld -### foo.o -m i386pep --rpath foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
+RUN: ld.lld -### foo.o -m i386pep -rpath=foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
+RUN: ld.lld -### foo.o -m i386pep --rpath=foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
+WARN_RPATH: warning: parameter -{{-?}}rpath has no effect on PE/COFF targets
diff --git a/lld/test/wasm/archive-local-sym.s b/lld/test/wasm/archive-local-sym.s
new file mode 100644
index 000000000000000..53e95b9bbb10567
--- /dev/null
+++ b/lld/test/wasm/archive-local-sym.s
@@ -0,0 +1,24 @@
+## Test that local symbols in archive files are ignored.
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t/foo.o %t/foo.s
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t/main.o %t/main.s
+# RUN: rm -f %t/libfoo.a
+# RUN: llvm-ar rcs %t/libfoo.a %t/foo.o
+# RUN: not wasm-ld %t/libfoo.a %t/main.o -o out.wasm 2>&1 | FileCheck %s
+
+#--- main.s
+
+.functype foo () -> ()
+
+.globl _start
+_start:
+  .functype _start () -> ()
+  call foo
+# CHECK: main.o: undefined symbol: foo
+  end_function
+
+#--- foo.s
+
+foo:
+  .functype foo () -> ()
+  end_function
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index 706ee25d5aae278..de8e707ab2b497d 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -392,7 +392,7 @@ void ObjFile::parseLazy() {
                     << wasmObj.get() << "\n");
   for (const SymbolRef &sym : wasmObj->symbols()) {
     const WasmSymbol &wasmSym = wasmObj->getWasmSymbol(sym.getRawDataRefImpl());
-    if (!wasmSym.isDefined())
+    if (wasmSym.isUndefined() || wasmSym.isBindingLocal())
       continue;
     symtab->addLazy(wasmSym.Info.Name, this);
     // addLazy() may trigger this->extract() if an existing symbol is an
diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp
index 6a66a29d2498c53..6beef81d39a3818 100644
--- a/lld/wasm/Writer.cpp
+++ b/lld/wasm/Writer.cpp
@@ -138,6 +138,11 @@ void Writer::calculateCustomSections() {
       // Exclude COMDAT sections that are not selected for inclusion
       if (section->discarded)
         continue;
+      // Ignore empty custom sections.  In particular objcopy/strip will
+      // sometimes replace stripped sections with empty custom sections to
+      // avoid section re-numbering.
+      if (section->getSize() == 0)
+        continue;
       StringRef name = section->name;
       // These custom sections are known the linker and synthesized rather than
       // blindly copied.
diff --git a/lldb/include/lldb/API/SBProcess.h b/lldb/include/lldb/API/SBProcess.h
index 778be7958399010..1624e02070b1b21 100644
--- a/lldb/include/lldb/API/SBProcess.h
+++ b/lldb/include/lldb/API/SBProcess.h
@@ -586,6 +586,7 @@ class LLDB_API SBProcess {
   friend class SBBreakpointCallbackBaton;
   friend class SBBreakpointLocation;
   friend class SBCommandInterpreter;
+  friend class SBSaveCoreOptions;
   friend class SBDebugger;
   friend class SBExecutionContext;
   friend class SBFunction;
diff --git a/lldb/include/lldb/API/SBSaveCoreOptions.h b/lldb/include/lldb/API/SBSaveCoreOptions.h
index 75506fd752e7621..ba48ba5eaea5a0b 100644
--- a/lldb/include/lldb/API/SBSaveCoreOptions.h
+++ b/lldb/include/lldb/API/SBSaveCoreOptions.h
@@ -10,6 +10,10 @@
 #define LLDB_API_SBSAVECOREOPTIONS_H
 
 #include "lldb/API/SBDefines.h"
+#include "lldb/API/SBError.h"
+#include "lldb/API/SBFileSpec.h"
+#include "lldb/API/SBProcess.h"
+#include "lldb/API/SBThread.h"
 
 namespace lldb {
 
@@ -53,6 +57,29 @@ class LLDB_API SBSaveCoreOptions {
   /// \return The output file spec.
   SBFileSpec GetOutputFile() const;
 
+  /// Set the process to save, or unset if supplied with a default constructed
+  /// process.
+  ///
+  /// \param process The process to save.
+  /// \return Success if process was set, otherwise an error
+  /// \note This will clear all process specific options if a different process
+  /// is specified than the current set process, either explicitly from this
+  /// api, or implicitly from any function that requires a process.
+  SBError SetProcess(lldb::SBProcess process);
+
+  /// Add a thread to save in the core file.
+  ///
+  /// \param thread The thread to save.
+  /// \note This will set the process if it is not already set, or return
+  /// and error if the SBThread is not from the set process.
+  SBError AddThread(lldb::SBThread thread);
+
+  /// Remove a thread from the list of threads to save.
+  ///
+  /// \param thread The thread to remove.
+  /// \return True if the thread was removed, false if it was not in the list.
+  bool RemoveThread(lldb::SBThread thread);
+
   /// Reset all options.
   void Clear();
 
diff --git a/lldb/include/lldb/API/SBThread.h b/lldb/include/lldb/API/SBThread.h
index dcf6aa9d5424e85..f8ae627da5acee0 100644
--- a/lldb/include/lldb/API/SBThread.h
+++ b/lldb/include/lldb/API/SBThread.h
@@ -233,6 +233,7 @@ class LLDB_API SBThread {
   friend class SBBreakpoint;
   friend class SBBreakpointLocation;
   friend class SBBreakpointCallbackBaton;
+  friend class SBSaveCoreOptions;
   friend class SBExecutionContext;
   friend class SBFrame;
   friend class SBProcess;
@@ -253,6 +254,8 @@ class LLDB_API SBThread {
   SBError ResumeNewPlan(lldb_private::ExecutionContext &exe_ctx,
                         lldb_private::ThreadPlan *new_plan);
 
+  lldb::ThreadSP GetSP() const;
+
   lldb::ExecutionContextRefSP m_opaque_sp;
 
   lldb_private::Thread *operator->();
diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h
index a23f834f471fb34..e4e0c3eea67f8c0 100644
--- a/lldb/include/lldb/Core/PluginManager.h
+++ b/lldb/include/lldb/Core/PluginManager.h
@@ -194,7 +194,7 @@ class PluginManager {
   GetObjectFileCreateMemoryCallbackForPluginName(llvm::StringRef name);
 
   static Status SaveCore(const lldb::ProcessSP &process_sp,
-                         const lldb_private::SaveCoreOptions &core_options);
+                         lldb_private::SaveCoreOptions &core_options);
 
   // ObjectContainer
   static bool RegisterPlugin(
diff --git a/lldb/include/lldb/Host/Socket.h b/lldb/include/lldb/Host/Socket.h
index 4dad88e78faa112..573c881f727d8f9 100644
--- a/lldb/include/lldb/Host/Socket.h
+++ b/lldb/include/lldb/Host/Socket.h
@@ -60,6 +60,10 @@ class Socket : public IOObject {
 
   ~Socket() override;
 
+  static const char *FindSchemeByProtocol(const SocketProtocol protocol);
+  static bool FindProtocolByScheme(const char *scheme,
+                                   SocketProtocol &protocol);
+
   static llvm::Error Initialize();
   static void Terminate();
 
diff --git a/lldb/include/lldb/Symbol/CompileUnit.h b/lldb/include/lldb/Symbol/CompileUnit.h
index c20a37e32830755..c5bb080d2118490 100644
--- a/lldb/include/lldb/Symbol/CompileUnit.h
+++ b/lldb/include/lldb/Symbol/CompileUnit.h
@@ -19,11 +19,13 @@
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-forward.h"
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 
 namespace lldb_private {
+
 /// \class CompileUnit CompileUnit.h "lldb/Symbol/CompileUnit.h"
 /// A class that describes a compilation unit.
 ///
@@ -389,10 +391,15 @@ class CompileUnit : public std::enable_shared_from_this<CompileUnit>,
   ///     A SymbolContext list class that will get any matching
   ///     entries appended to.
   ///
+  /// \param[in] realpath_prefixes
+  ///     Paths that start with one of the prefixes in this list will be
+  ///     realpath'ed to resolve any symlinks.
+  ///
   /// \see enum SymbolContext::Scope
   void ResolveSymbolContext(const SourceLocationSpec &src_location_spec,
                             lldb::SymbolContextItem resolve_scope,
-                            SymbolContextList &sc_list);
+                            SymbolContextList &sc_list,
+                            RealpathPrefixes *realpath_prefixes = nullptr);
 
   /// Get whether compiler optimizations were enabled for this compile unit
   ///
diff --git a/lldb/include/lldb/Symbol/SaveCoreOptions.h b/lldb/include/lldb/Symbol/SaveCoreOptions.h
index 583bc1f483d043a..f4fed4676fa4ae1 100644
--- a/lldb/include/lldb/Symbol/SaveCoreOptions.h
+++ b/lldb/include/lldb/Symbol/SaveCoreOptions.h
@@ -15,6 +15,7 @@
 
 #include <optional>
 #include <string>
+#include <unordered_set>
 
 namespace lldb_private {
 
@@ -32,13 +33,25 @@ class SaveCoreOptions {
   void SetOutputFile(lldb_private::FileSpec file);
   const std::optional<lldb_private::FileSpec> GetOutputFile() const;
 
+  Status SetProcess(lldb::ProcessSP process_sp);
+
+  Status AddThread(lldb::ThreadSP thread_sp);
+  bool RemoveThread(lldb::ThreadSP thread_sp);
+  bool ShouldThreadBeSaved(lldb::tid_t tid) const;
+
+  Status EnsureValidConfiguration(lldb::ProcessSP process_sp) const;
+
   void Clear();
 
 private:
+  void ClearProcessSpecificData();
+
   std::optional<std::string> m_plugin_name;
   std::optional<lldb_private::FileSpec> m_file;
   std::optional<lldb::SaveCoreStyle> m_style;
+  lldb::ProcessSP m_process_sp;
+  std::unordered_set<lldb::tid_t> m_threads_to_save;
 };
 } // namespace lldb_private
 
-#endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_SaveCoreOPTIONS_H
+#endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_SAVECOREOPTIONS_H
diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h
index cf16fbc812aa48d..9cf6f50558570bc 100644
--- a/lldb/include/lldb/Target/Process.h
+++ b/lldb/include/lldb/Target/Process.h
@@ -738,9 +738,15 @@ class Process : public std::enable_shared_from_this<Process>,
   /// Helper function for Process::SaveCore(...) that calculates the address
   /// ranges that should be saved. This allows all core file plug-ins to save
   /// consistent memory ranges given a \a core_style.
-  Status CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
+  Status CalculateCoreFileSaveRanges(const SaveCoreOptions &core_options,
                                      CoreFileMemoryRanges &ranges);
 
+  /// Helper function for Process::SaveCore(...) that calculates the thread list
+  /// based upon options set within a given \a core_options object.
+  /// \note If there is no thread list defined, all threads will be saved.
+  std::vector<lldb::ThreadSP>
+  CalculateCoreFileThreadList(const SaveCoreOptions &core_options);
+
 protected:
   virtual JITLoaderList &GetJITLoaders();
 
diff --git a/lldb/include/lldb/Target/Statistics.h b/lldb/include/lldb/Target/Statistics.h
index 35bd7f8a66e055e..5193d099a5494d8 100644
--- a/lldb/include/lldb/Target/Statistics.h
+++ b/lldb/include/lldb/Target/Statistics.h
@@ -10,6 +10,7 @@
 #define LLDB_TARGET_STATISTICS_H
 
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/lldb-forward.h"
 #include "llvm/ADT/StringMap.h"
@@ -184,6 +185,8 @@ class TargetStats {
   void SetFirstPrivateStopTime();
   void SetFirstPublicStopTime();
   void IncreaseSourceMapDeduceCount();
+  void IncreaseSourceRealpathAttemptCount(uint32_t count);
+  void IncreaseSourceRealpathCompatibleCount(uint32_t count);
 
   StatsDuration &GetCreateTime() { return m_create_time; }
   StatsSuccessFail &GetExpressionStats() { return m_expr_eval; }
@@ -198,6 +201,8 @@ class TargetStats {
   StatsSuccessFail m_frame_var{"frameVariable"};
   std::vector<intptr_t> m_module_identifiers;
   uint32_t m_source_map_deduce_count = 0;
+  uint32_t m_source_realpath_attempt_count = 0;
+  uint32_t m_source_realpath_compatible_count = 0;
   void CollectStats(Target &target);
 };
 
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index 119dff4d498199f..7f4d607f5427dff 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -34,6 +34,7 @@
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/Broadcaster.h"
 #include "lldb/Utility/LLDBAssert.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/Timeout.h"
 #include "lldb/lldb-public.h"
 
@@ -117,6 +118,8 @@ class TargetProperties : public Properties {
 
   InlineStrategy GetInlineStrategy() const;
 
+  RealpathPrefixes GetSourceRealpathPrefixes() const;
+
   llvm::StringRef GetArg0() const;
 
   void SetArg0(llvm::StringRef arg);
diff --git a/lldb/include/lldb/Utility/FileSpecList.h b/lldb/include/lldb/Utility/FileSpecList.h
index 6eb3bb9971f13ad..d091a9246e08277 100644
--- a/lldb/include/lldb/Utility/FileSpecList.h
+++ b/lldb/include/lldb/Utility/FileSpecList.h
@@ -64,10 +64,16 @@ class SupportFileList {
   /// \param[in] file
   ///     The file specification to search for.
   ///
+  /// \param[in] realpath_prefixes
+  ///     Paths that start with one of the prefixes in this list will be
+  ///     realpath'ed to resolve any symlinks.
+  ///
   /// \return
   ///     The index of the file that matches \a file if it is found,
   ///     else UINT32_MAX is returned.
-  size_t FindCompatibleIndex(size_t idx, const FileSpec &file) const;
+  size_t
+  FindCompatibleIndex(size_t idx, const FileSpec &file,
+                      RealpathPrefixes *realpath_prefixes = nullptr) const;
 
   template <class... Args> void EmplaceBack(Args &&...args) {
     m_files.push_back(
diff --git a/lldb/include/lldb/Utility/RealpathPrefixes.h b/lldb/include/lldb/Utility/RealpathPrefixes.h
new file mode 100644
index 000000000000000..daa2a479d6a2469
--- /dev/null
+++ b/lldb/include/lldb/Utility/RealpathPrefixes.h
@@ -0,0 +1,77 @@
+//===-- RealpathPrefixes.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_UTILITY_REALPATHPREFIXES_H
+#define LLDB_UTILITY_REALPATHPREFIXES_H
+
+#include "lldb/lldb-forward.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace lldb_private {
+
+class RealpathPrefixes {
+public:
+  /// \param[in] file_spec_list
+  ///     Prefixes are obtained from FileSpecList, through FileSpec::GetPath(),
+  ///     which ensures that the paths are normalized. For example:
+  ///     "./foo/.." -> ""
+  ///     "./foo/../bar" -> "bar"
+  ///
+  /// \param[in] fs
+  ///     An optional filesystem to use for realpath'ing. If not set, the real
+  ///     filesystem will be used.
+  explicit RealpathPrefixes(const FileSpecList &file_spec_list,
+                            llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs =
+                                llvm::vfs::getRealFileSystem());
+
+  std::optional<FileSpec> ResolveSymlinks(const FileSpec &file_spec);
+
+  // If/when Statistics.h/cpp is moved into Utility, we can remove these
+  // methods, hold a (weak) pointer to `TargetStats` and directly increment
+  // on that object.
+  void IncreaseSourceRealpathAttemptCount() {
+    ++m_source_realpath_attempt_count;
+  }
+  uint32_t GetSourceRealpathAttemptCount() const {
+    return m_source_realpath_attempt_count;
+  }
+  void IncreaseSourceRealpathCompatibleCount() {
+    ++m_source_realpath_compatible_count;
+  }
+  uint32_t GetSourceRealpathCompatibleCount() const {
+    return m_source_realpath_compatible_count;
+  }
+
+private:
+  // Paths that start with one of the prefixes in this list will be realpath'ed
+  // to resolve any symlinks.
+  //
+  // Wildcard prefixes:
+  // - "" (empty string) will match all paths.
+  // - "/" will match all absolute paths.
+  std::vector<std::string> m_prefixes;
+
+  // The filesystem to use for realpath'ing.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> m_fs;
+
+  // The optional Target instance to gather statistics.
+  lldb::TargetWP m_target;
+
+  // Statistics that we temprarily hold here, to be gathered into TargetStats
+  uint32_t m_source_realpath_attempt_count = 0;
+  uint32_t m_source_realpath_compatible_count = 0;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_UTILITY_REALPATHPREFIXES_H
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 1024501e05bcac7..337eff696fcf3f5 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -175,6 +175,7 @@ class Queue;
 class QueueImpl;
 class QueueItem;
 class REPL;
+class RealpathPrefixes;
 class RecognizedStackFrame;
 class RegisterCheckpoint;
 class RegisterContext;
diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h
index 87c5ff8d22fb682..b3c8cda899b95ef 100644
--- a/lldb/include/lldb/lldb-private-interfaces.h
+++ b/lldb/include/lldb/lldb-private-interfaces.h
@@ -57,7 +57,7 @@ typedef ObjectFile *(*ObjectFileCreateMemoryInstance)(
     const lldb::ModuleSP &module_sp, lldb::WritableDataBufferSP data_sp,
     const lldb::ProcessSP &process_sp, lldb::addr_t offset);
 typedef bool (*ObjectFileSaveCore)(const lldb::ProcessSP &process_sp,
-                                   const lldb_private::SaveCoreOptions &options,
+                                   lldb_private::SaveCoreOptions &options,
                                    Status &error);
 typedef EmulateInstruction *(*EmulateInstructionCreateInstance)(
     const ArchSpec &arch, InstructionType inst_type);
diff --git a/lldb/include/lldb/lldb-private-types.h b/lldb/include/lldb/lldb-private-types.h
index 055eea9f6456d6b..b82a2b8aa057443 100644
--- a/lldb/include/lldb/lldb-private-types.h
+++ b/lldb/include/lldb/lldb-private-types.h
@@ -12,6 +12,7 @@
 #include "lldb/lldb-private.h"
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
 
 #include <type_traits>
 
@@ -26,6 +27,8 @@ class Platform;
 class ExecutionContext;
 class RegisterFlags;
 
+typedef llvm::SmallString<256> PathSmallString;
+
 typedef llvm::sys::DynamicLibrary (*LoadPluginCallbackType)(
     const lldb::DebuggerSP &debugger_sp, const FileSpec &spec, Status &error);
 
diff --git a/lldb/source/API/SBSaveCoreOptions.cpp b/lldb/source/API/SBSaveCoreOptions.cpp
index 19ca83f932bcf14..2cd431611ef558c 100644
--- a/lldb/source/API/SBSaveCoreOptions.cpp
+++ b/lldb/source/API/SBSaveCoreOptions.cpp
@@ -7,8 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/API/SBSaveCoreOptions.h"
-#include "lldb/API/SBError.h"
-#include "lldb/API/SBFileSpec.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Symbol/SaveCoreOptions.h"
 #include "lldb/Utility/Instrumentation.h"
@@ -77,6 +75,21 @@ lldb::SaveCoreStyle SBSaveCoreOptions::GetStyle() const {
   return m_opaque_up->GetStyle();
 }
 
+SBError SBSaveCoreOptions::SetProcess(lldb::SBProcess process) {
+  LLDB_INSTRUMENT_VA(this, process);
+  return m_opaque_up->SetProcess(process.GetSP());
+}
+
+SBError SBSaveCoreOptions::AddThread(lldb::SBThread thread) {
+  LLDB_INSTRUMENT_VA(this, thread);
+  return m_opaque_up->AddThread(thread.GetSP());
+}
+
+bool SBSaveCoreOptions::RemoveThread(lldb::SBThread thread) {
+  LLDB_INSTRUMENT_VA(this, thread);
+  return m_opaque_up->RemoveThread(thread.GetSP());
+}
+
 void SBSaveCoreOptions::Clear() {
   LLDB_INSTRUMENT_VA(this);
   m_opaque_up->Clear();
diff --git a/lldb/source/API/SBThread.cpp b/lldb/source/API/SBThread.cpp
index bda981041064ff7..786f62bd66d520a 100644
--- a/lldb/source/API/SBThread.cpp
+++ b/lldb/source/API/SBThread.cpp
@@ -1337,6 +1337,8 @@ bool SBThread::SafeToCallFunctions() {
   return true;
 }
 
+lldb::ThreadSP SBThread::GetSP() const { return m_opaque_sp->GetThreadSP(); }
+
 lldb_private::Thread *SBThread::operator->() {
   return get();
 }
diff --git a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp
index 16c4ee1b88d1620..508754082cae8a5 100644
--- a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp
@@ -15,6 +15,7 @@
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/StreamString.h"
 #include <optional>
 
@@ -290,16 +291,25 @@ Searcher::CallbackReturn BreakpointResolverFileLine::SearchCallback(
   const uint32_t line = m_location_spec.GetLine().value_or(0);
   const std::optional<uint16_t> column = m_location_spec.GetColumn();
 
+  Target &target = GetBreakpoint()->GetTarget();
+  RealpathPrefixes realpath_prefixes = target.GetSourceRealpathPrefixes();
+
   const size_t num_comp_units = context.module_sp->GetNumCompileUnits();
   for (size_t i = 0; i < num_comp_units; i++) {
     CompUnitSP cu_sp(context.module_sp->GetCompileUnitAtIndex(i));
     if (cu_sp) {
       if (filter.CompUnitPasses(*cu_sp))
         cu_sp->ResolveSymbolContext(m_location_spec, eSymbolContextEverything,
-                                    sc_list);
+                                    sc_list, &realpath_prefixes);
     }
   }
 
+  // Gather stats into the Target
+  target.GetStatistics().IncreaseSourceRealpathAttemptCount(
+      realpath_prefixes.GetSourceRealpathAttemptCount());
+  target.GetStatistics().IncreaseSourceRealpathCompatibleCount(
+      realpath_prefixes.GetSourceRealpathCompatibleCount());
+
   FilterContexts(sc_list);
 
   DeduceSourceMapping(sc_list);
diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp
index 01bee8680b7ba51..f243807df509efd 100644
--- a/lldb/source/Core/PluginManager.cpp
+++ b/lldb/source/Core/PluginManager.cpp
@@ -702,7 +702,7 @@ PluginManager::GetObjectFileCreateMemoryCallbackForPluginName(
 }
 
 Status PluginManager::SaveCore(const lldb::ProcessSP &process_sp,
-                               const lldb_private::SaveCoreOptions &options) {
+                               lldb_private::SaveCoreOptions &options) {
   Status error;
   if (!options.GetOutputFile()) {
     error.SetErrorString("No output file specified");
@@ -714,6 +714,10 @@ Status PluginManager::SaveCore(const lldb::ProcessSP &process_sp,
     return error;
   }
 
+  error = options.EnsureValidConfiguration(process_sp);
+  if (error.Fail())
+    return error;
+
   if (!options.GetPluginName().has_value()) {
     // Try saving core directly from the process plugin first.
     llvm::Expected<bool> ret =
diff --git a/lldb/source/Host/common/Socket.cpp b/lldb/source/Host/common/Socket.cpp
index f9911cf136cbd09..7364a12280cfdd0 100644
--- a/lldb/source/Host/common/Socket.cpp
+++ b/lldb/source/Host/common/Socket.cpp
@@ -70,6 +70,38 @@ static bool IsInterrupted() {
 #endif
 }
 
+struct SocketScheme {
+  const char *m_scheme;
+  const Socket::SocketProtocol m_protocol;
+};
+
+static SocketScheme socket_schemes[] = {
+    {"tcp", Socket::ProtocolTcp},
+    {"udp", Socket::ProtocolUdp},
+    {"unix", Socket::ProtocolUnixDomain},
+    {"unix-abstract", Socket::ProtocolUnixAbstract},
+};
+
+const char *
+Socket::FindSchemeByProtocol(const Socket::SocketProtocol protocol) {
+  for (auto s : socket_schemes) {
+    if (s.m_protocol == protocol)
+      return s.m_scheme;
+  }
+  return nullptr;
+}
+
+bool Socket::FindProtocolByScheme(const char *scheme,
+                                  Socket::SocketProtocol &protocol) {
+  for (auto s : socket_schemes) {
+    if (!strcmp(s.m_scheme, scheme)) {
+      protocol = s.m_protocol;
+      return true;
+    }
+  }
+  return false;
+}
+
 Socket::Socket(SocketProtocol protocol, bool should_close,
                bool child_processes_inherit)
     : IOObject(eFDTypeSocket), m_protocol(protocol),
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h b/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
index da2b1a15f74614a..79ed74b79f04fd6 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
@@ -264,10 +264,6 @@ class SemaSourceWithPriorities : public clang::ExternalSemaSource {
 
   ~SemaSourceWithPriorities() override;
 
-  void addSource(clang::ExternalSemaSource &source) {
-    Sources.push_back(&source);
-  }
-
   //===--------------------------------------------------------------------===//
   // ExternalASTSource.
   //===--------------------------------------------------------------------===//
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index ce095bcc48374b2..22ece4f4dacf790 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -6347,22 +6347,24 @@ struct segment_vmaddr {
 // are some multiple passes over the image list while calculating
 // everything.
 
-static offset_t CreateAllImageInfosPayload(
-    const lldb::ProcessSP &process_sp, offset_t initial_file_offset,
-    StreamString &all_image_infos_payload, SaveCoreStyle core_style) {
+static offset_t
+CreateAllImageInfosPayload(const lldb::ProcessSP &process_sp,
+                           offset_t initial_file_offset,
+                           StreamString &all_image_infos_payload,
+                           lldb_private::SaveCoreOptions &options) {
   Target &target = process_sp->GetTarget();
   ModuleList modules = target.GetImages();
 
   // stack-only corefiles have no reason to include binaries that
   // are not executing; we're trying to make the smallest corefile
   // we can, so leave the rest out.
-  if (core_style == SaveCoreStyle::eSaveCoreStackOnly)
+  if (options.GetStyle() == SaveCoreStyle::eSaveCoreStackOnly)
     modules.Clear();
 
   std::set<std::string> executing_uuids;
-  ThreadList &thread_list(process_sp->GetThreadList());
-  for (uint32_t i = 0; i < thread_list.GetSize(); i++) {
-    ThreadSP thread_sp = thread_list.GetThreadAtIndex(i);
+  std::vector<ThreadSP> thread_list =
+      process_sp->CalculateCoreFileThreadList(options);
+  for (const ThreadSP &thread_sp : thread_list) {
     uint32_t stack_frame_count = thread_sp->GetStackFrameCount();
     for (uint32_t j = 0; j < stack_frame_count; j++) {
       StackFrameSP stack_frame_sp = thread_sp->GetStackFrameAtIndex(j);
@@ -6520,16 +6522,17 @@ struct page_object {
 };
 
 bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
-                               const lldb_private::SaveCoreOptions &options,
+                               lldb_private::SaveCoreOptions &options,
                                Status &error) {
-  auto core_style = options.GetStyle();
-  if (core_style == SaveCoreStyle::eSaveCoreUnspecified)
-    core_style = SaveCoreStyle::eSaveCoreDirtyOnly;
   // The FileSpec and Process are already checked in PluginManager::SaveCore.
   assert(options.GetOutputFile().has_value());
   assert(process_sp);
   const FileSpec outfile = options.GetOutputFile().value();
 
+  // MachO defaults to dirty pages
+  if (options.GetStyle() == SaveCoreStyle::eSaveCoreUnspecified)
+    options.SetStyle(eSaveCoreDirtyOnly);
+
   Target &target = process_sp->GetTarget();
   const ArchSpec target_arch = target.GetArchitecture();
   const llvm::Triple &target_triple = target_arch.GetTriple();
@@ -6559,7 +6562,7 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
 
     if (make_core) {
       Process::CoreFileMemoryRanges core_ranges;
-      error = process_sp->CalculateCoreFileSaveRanges(core_style, core_ranges);
+      error = process_sp->CalculateCoreFileSaveRanges(options, core_ranges);
       if (error.Success()) {
         const uint32_t addr_byte_size = target_arch.GetAddressByteSize();
         const ByteOrder byte_order = target_arch.GetByteOrder();
@@ -6730,8 +6733,8 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
             std::make_shared<StructuredData::Dictionary>());
         StructuredData::ArraySP threads(
             std::make_shared<StructuredData::Array>());
-        for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-          ThreadSP thread_sp(thread_list.GetThreadAtIndex(thread_idx));
+        for (const ThreadSP &thread_sp :
+             process_sp->CalculateCoreFileThreadList(options)) {
           StructuredData::DictionarySP thread(
               std::make_shared<StructuredData::Dictionary>());
           thread->AddIntegerItem("thread_id", thread_sp->GetID());
@@ -6754,7 +6757,7 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
         all_image_infos_lcnote_up->payload_file_offset = file_offset;
         file_offset = CreateAllImageInfosPayload(
             process_sp, file_offset, all_image_infos_lcnote_up->payload,
-            core_style);
+            options);
         lc_notes.push_back(std::move(all_image_infos_lcnote_up));
 
         // Add LC_NOTE load commands
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
index e7af90e28bc4b7e..27bc237aaac48d8 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
@@ -62,7 +62,7 @@ class ObjectFileMachO : public lldb_private::ObjectFile {
                                         lldb_private::ModuleSpecList &specs);
 
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::SaveCoreOptions &options,
+                       lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
   static bool MagicBytesMatch(lldb::DataBufferSP data_sp, lldb::addr_t offset,
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
index de212c6b20da7eb..c0cc3af638a7772 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
+++ b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
@@ -69,10 +69,9 @@ Status MinidumpFileBuilder::AddHeaderAndCalculateDirectories() {
     m_expected_directories += 9;
 
   // Go through all of the threads and check for exceptions.
-  lldb_private::ThreadList thread_list = m_process_sp->GetThreadList();
-  const uint32_t num_threads = thread_list.GetSize();
-  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-    ThreadSP thread_sp(thread_list.GetThreadAtIndex(thread_idx));
+  std::vector<lldb::ThreadSP> threads =
+      m_process_sp->CalculateCoreFileThreadList(m_save_core_options);
+  for (const ThreadSP &thread_sp : threads) {
     StopInfoSP stop_info_sp = thread_sp->GetStopInfo();
     if (stop_info_sp) {
       const StopReason &stop_reason = stop_info_sp->GetStopReason();
@@ -588,12 +587,13 @@ Status MinidumpFileBuilder::FixThreadStacks() {
 
 Status MinidumpFileBuilder::AddThreadList() {
   constexpr size_t minidump_thread_size = sizeof(llvm::minidump::Thread);
-  lldb_private::ThreadList thread_list = m_process_sp->GetThreadList();
+  std::vector<ThreadSP> thread_list =
+      m_process_sp->CalculateCoreFileThreadList(m_save_core_options);
 
   // size of the entire thread stream consists of:
   // number of threads and threads array
   size_t thread_stream_size = sizeof(llvm::support::ulittle32_t) +
-                              thread_list.GetSize() * minidump_thread_size;
+                              thread_list.size() * minidump_thread_size;
   // save for the ability to set up RVA
   size_t size_before = GetCurrentDataEndOffset();
   Status error;
@@ -602,17 +602,15 @@ Status MinidumpFileBuilder::AddThreadList() {
     return error;
 
   llvm::support::ulittle32_t thread_count =
-      static_cast<llvm::support::ulittle32_t>(thread_list.GetSize());
+      static_cast<llvm::support::ulittle32_t>(thread_list.size());
   m_data.AppendData(&thread_count, sizeof(llvm::support::ulittle32_t));
 
   // Take the offset after the thread count.
   m_thread_list_start = GetCurrentDataEndOffset();
   DataBufferHeap helper_data;
 
-  const uint32_t num_threads = thread_list.GetSize();
   Log *log = GetLog(LLDBLog::Object);
-  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-    ThreadSP thread_sp(thread_list.GetThreadAtIndex(thread_idx));
+  for (const ThreadSP &thread_sp : thread_list) {
     RegisterContextSP reg_ctx_sp(thread_sp->GetRegisterContext());
 
     if (!reg_ctx_sp) {
@@ -650,7 +648,7 @@ Status MinidumpFileBuilder::AddThreadList() {
     m_tid_to_reg_ctx[thread_sp->GetID()] = thread_context_memory_locator;
 
     LLDB_LOGF(log, "AddThreadList for thread %d: thread_context %zu bytes",
-              thread_idx, thread_context.size());
+              thread_sp->GetIndexID(), thread_context.size());
     helper_data.AppendData(thread_context.data(), thread_context.size());
 
     llvm::minidump::Thread t;
@@ -674,11 +672,10 @@ Status MinidumpFileBuilder::AddThreadList() {
 }
 
 Status MinidumpFileBuilder::AddExceptions() {
-  lldb_private::ThreadList thread_list = m_process_sp->GetThreadList();
+  std::vector<ThreadSP> thread_list =
+      m_process_sp->CalculateCoreFileThreadList(m_save_core_options);
   Status error;
-  const uint32_t num_threads = thread_list.GetSize();
-  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-    ThreadSP thread_sp(thread_list.GetThreadAtIndex(thread_idx));
+  for (const ThreadSP &thread_sp : thread_list) {
     StopInfoSP stop_info_sp = thread_sp->GetStopInfo();
     bool add_exception = false;
     if (stop_info_sp) {
@@ -819,7 +816,7 @@ Status MinidumpFileBuilder::AddLinuxFileStreams() {
   return error;
 }
 
-Status MinidumpFileBuilder::AddMemoryList(SaveCoreStyle core_style) {
+Status MinidumpFileBuilder::AddMemoryList() {
   Status error;
 
   // We first save the thread stacks to ensure they fit in the first UINT32_MAX
@@ -828,18 +825,26 @@ Status MinidumpFileBuilder::AddMemoryList(SaveCoreStyle core_style) {
   // in accessible with a 32 bit offset.
   Process::CoreFileMemoryRanges ranges_32;
   Process::CoreFileMemoryRanges ranges_64;
-  error = m_process_sp->CalculateCoreFileSaveRanges(
-      SaveCoreStyle::eSaveCoreStackOnly, ranges_32);
+  Process::CoreFileMemoryRanges all_core_memory_ranges;
+  error = m_process_sp->CalculateCoreFileSaveRanges(m_save_core_options,
+                                                    all_core_memory_ranges);
   if (error.Fail())
     return error;
 
-  // Calculate totalsize including the current offset.
+  // Start by saving all of the stacks and ensuring they fit under the 32b
+  // limit.
   uint64_t total_size = GetCurrentDataEndOffset();
-  total_size += ranges_32.size() * sizeof(llvm::minidump::MemoryDescriptor);
-  std::unordered_set<addr_t> stack_start_addresses;
-  for (const auto &core_range : ranges_32) {
-    stack_start_addresses.insert(core_range.range.start());
-    total_size += core_range.range.size();
+  auto iterator = all_core_memory_ranges.begin();
+  while (iterator != all_core_memory_ranges.end()) {
+    if (m_saved_stack_ranges.count(iterator->range.start()) > 0) {
+      // We don't save stacks twice.
+      ranges_32.push_back(*iterator);
+      total_size +=
+          iterator->range.size() + sizeof(llvm::minidump::MemoryDescriptor);
+      iterator = all_core_memory_ranges.erase(iterator);
+    } else {
+      iterator++;
+    }
   }
 
   if (total_size >= UINT32_MAX) {
@@ -849,14 +854,6 @@ Status MinidumpFileBuilder::AddMemoryList(SaveCoreStyle core_style) {
     return error;
   }
 
-  Process::CoreFileMemoryRanges all_core_memory_ranges;
-  if (core_style != SaveCoreStyle::eSaveCoreStackOnly) {
-    error = m_process_sp->CalculateCoreFileSaveRanges(core_style,
-                                                      all_core_memory_ranges);
-    if (error.Fail())
-      return error;
-  }
-
   // After saving the stacks, we start packing as much as we can into 32b.
   // We apply a generous padding here so that the Directory, MemoryList and
   // Memory64List sections all begin in 32b addressable space.
@@ -864,16 +861,13 @@ Status MinidumpFileBuilder::AddMemoryList(SaveCoreStyle core_style) {
   // All core memeroy ranges will either container nothing on stacks only
   // or all the memory ranges including stacks
   if (!all_core_memory_ranges.empty())
-    total_size +=
-        256 + (all_core_memory_ranges.size() - stack_start_addresses.size()) *
-                  sizeof(llvm::minidump::MemoryDescriptor_64);
+    total_size += 256 + (all_core_memory_ranges.size() *
+                         sizeof(llvm::minidump::MemoryDescriptor_64));
 
   for (const auto &core_range : all_core_memory_ranges) {
     const addr_t range_size = core_range.range.size();
-    if (stack_start_addresses.count(core_range.range.start()) > 0)
-      // Don't double save stacks.
-      continue;
-
+    // We don't need to check for stacks here because we already removed them
+    // from all_core_memory_ranges.
     if (total_size + range_size < UINT32_MAX) {
       ranges_32.push_back(core_range);
       total_size += range_size;
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
index 20564e0661f2a29..762de83db5a39ce 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
+++ b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
@@ -75,8 +75,10 @@ lldb_private::Status WriteString(const std::string &to_write,
 class MinidumpFileBuilder {
 public:
   MinidumpFileBuilder(lldb::FileUP &&core_file,
-                      const lldb::ProcessSP &process_sp)
-      : m_process_sp(process_sp), m_core_file(std::move(core_file)){};
+                      const lldb::ProcessSP &process_sp,
+                      lldb_private::SaveCoreOptions &save_core_options)
+      : m_process_sp(process_sp), m_core_file(std::move(core_file)),
+        m_save_core_options(save_core_options){}
 
   MinidumpFileBuilder(const MinidumpFileBuilder &) = delete;
   MinidumpFileBuilder &operator=(const MinidumpFileBuilder &) = delete;
@@ -103,7 +105,7 @@ class MinidumpFileBuilder {
   // Add Exception streams for any threads that stopped with exceptions.
   lldb_private::Status AddExceptions();
   // Add MemoryList stream, containing dumps of important memory segments
-  lldb_private::Status AddMemoryList(lldb::SaveCoreStyle core_style);
+  lldb_private::Status AddMemoryList();
   // Add MiscInfo stream, mainly providing ProcessId
   lldb_private::Status AddMiscInfo();
   // Add informative files about a Linux process
@@ -163,7 +165,9 @@ class MinidumpFileBuilder {
   // to duplicate it in the exception data.
   std::unordered_map<lldb::tid_t, llvm::minidump::LocationDescriptor>
       m_tid_to_reg_ctx;
+  std::unordered_set<lldb::addr_t> m_saved_stack_ranges;
   lldb::FileUP m_core_file;
+  lldb_private::SaveCoreOptions m_save_core_options;
 };
 
 #endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_MINIDUMP_MINIDUMPFILEBUILDER_H
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
index faa144bfb5f6a5b..0897895e6bc25da 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
+++ b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
@@ -56,16 +56,15 @@ size_t ObjectFileMinidump::GetModuleSpecifications(
 }
 
 bool ObjectFileMinidump::SaveCore(const lldb::ProcessSP &process_sp,
-                                  const lldb_private::SaveCoreOptions &options,
+                                  lldb_private::SaveCoreOptions &options,
                                   lldb_private::Status &error) {
   // Output file and process_sp are both checked in PluginManager::SaveCore.
   assert(options.GetOutputFile().has_value());
   assert(process_sp);
 
   // Minidump defaults to stacks only.
-  SaveCoreStyle core_style = options.GetStyle();
-  if (core_style == SaveCoreStyle::eSaveCoreUnspecified)
-    core_style = SaveCoreStyle::eSaveCoreStackOnly;
+  if (options.GetStyle() == SaveCoreStyle::eSaveCoreUnspecified)
+    options.SetStyle(SaveCoreStyle::eSaveCoreStackOnly);
 
   llvm::Expected<lldb::FileUP> maybe_core_file = FileSystem::Instance().Open(
       options.GetOutputFile().value(),
@@ -74,7 +73,8 @@ bool ObjectFileMinidump::SaveCore(const lldb::ProcessSP &process_sp,
     error = maybe_core_file.takeError();
     return false;
   }
-  MinidumpFileBuilder builder(std::move(maybe_core_file.get()), process_sp);
+  MinidumpFileBuilder builder(std::move(maybe_core_file.get()), process_sp,
+                              options);
 
   Log *log = GetLog(LLDBLog::Object);
   error = builder.AddHeaderAndCalculateDirectories();
@@ -121,7 +121,7 @@ bool ObjectFileMinidump::SaveCore(const lldb::ProcessSP &process_sp,
 
   // Note: add memory HAS to be the last thing we do. It can overflow into 64b
   // land and many RVA's only support 32b
-  error = builder.AddMemoryList(core_style);
+  error = builder.AddMemoryList();
   if (error.Fail()) {
     LLDB_LOGF(log, "AddMemoryList failed: %s", error.AsCString());
     return false;
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
index 0cd31a0e482d5f8..b76fcd0052a8a8c 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
+++ b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
@@ -55,7 +55,7 @@ class ObjectFileMinidump : public lldb_private::PluginInterface {
 
   // Saves dump in Minidump file format
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::SaveCoreOptions &options,
+                       lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
 private:
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index bda691ade8af025..9d01089745dfc91 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -355,7 +355,7 @@ size_t ObjectFilePECOFF::GetModuleSpecifications(
 }
 
 bool ObjectFilePECOFF::SaveCore(const lldb::ProcessSP &process_sp,
-                                const lldb_private::SaveCoreOptions &options,
+                                lldb_private::SaveCoreOptions &options,
                                 lldb_private::Status &error) {
   // Outfile and process_sp are validated by PluginManager::SaveCore
   assert(options.GetOutputFile().has_value());
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
index 2eb2b3b7745380d..8bccf3be3e5f636 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
@@ -82,7 +82,7 @@ class ObjectFilePECOFF : public lldb_private::ObjectFile {
                                         lldb_private::ModuleSpecList &specs);
 
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::SaveCoreOptions &options,
+                       lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
   static bool MagicBytesMatch(lldb::DataBufferSP data_sp);
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
index 6b8488990a97200..24ab881d92d85a7 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
@@ -21,8 +21,7 @@
 namespace lldb_private {
 
 bool SaveMiniDump(const lldb::ProcessSP &process_sp,
-                  const SaveCoreOptions &core_options,
-                  lldb_private::Status &error) {
+                  SaveCoreOptions &core_options, lldb_private::Status &error) {
   if (!process_sp)
     return false;
 #ifdef _WIN32
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
index 03c0ece30614318..2f8606d82c974b0 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
@@ -14,8 +14,7 @@
 namespace lldb_private {
 
 bool SaveMiniDump(const lldb::ProcessSP &process_sp,
-                  const SaveCoreOptions &core_options,
-                  lldb_private::Status &error);
+                  SaveCoreOptions &core_options, lldb_private::Status &error);
 
 } // namespace lldb_private
 
diff --git a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
index 2a83163884e168b..a3ee9dff572766b 100644
--- a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
+++ b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
@@ -13,6 +13,7 @@
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/Utility/ProcessInfo.h"
 
 #include "Plugins/Process/Utility/RegisterContextFreeBSD_i386.h"
 #include "Plugins/Process/Utility/RegisterContextFreeBSD_mips64.h"
@@ -318,6 +319,32 @@ Status ELFLinuxPrStatus::Parse(const DataExtractor &data,
   return error;
 }
 
+static struct compat_timeval
+copy_timespecs(const ProcessInstanceInfo::timespec &oth) {
+  using sec_t = decltype(compat_timeval::tv_sec);
+  using usec_t = decltype(compat_timeval::tv_usec);
+  return {static_cast<sec_t>(oth.tv_sec), static_cast<usec_t>(oth.tv_usec)};
+}
+
+std::optional<ELFLinuxPrStatus>
+ELFLinuxPrStatus::Populate(const lldb::ThreadSP &thread_sp) {
+  ELFLinuxPrStatus prstatus{};
+  prstatus.pr_pid = thread_sp->GetID();
+  lldb::ProcessSP process_sp = thread_sp->GetProcess();
+  ProcessInstanceInfo info;
+  if (!process_sp->GetProcessInfo(info))
+    return std::nullopt;
+
+  prstatus.pr_ppid = info.GetParentProcessID();
+  prstatus.pr_pgrp = info.GetProcessGroupID();
+  prstatus.pr_sid = info.GetProcessSessionID();
+  prstatus.pr_utime = copy_timespecs(info.GetUserTime());
+  prstatus.pr_stime = copy_timespecs(info.GetSystemTime());
+  prstatus.pr_cutime = copy_timespecs(info.GetCumulativeUserTime());
+  prstatus.pr_cstime = copy_timespecs(info.GetCumulativeSystemTime());
+  return prstatus;
+}
+
 // Parse PRPSINFO from NOTE entry
 ELFLinuxPrPsInfo::ELFLinuxPrPsInfo() {
   memset(this, 0, sizeof(ELFLinuxPrPsInfo));
@@ -394,6 +421,108 @@ Status ELFLinuxPrPsInfo::Parse(const DataExtractor &data,
   return error;
 }
 
+std::optional<ELFLinuxPrPsInfo>
+ELFLinuxPrPsInfo::Populate(const lldb::ProcessSP &process_sp) {
+  ProcessInstanceInfo info;
+  if (!process_sp->GetProcessInfo(info))
+    return std::nullopt;
+
+  return Populate(info, process_sp->GetState());
+}
+
+std::optional<ELFLinuxPrPsInfo>
+ELFLinuxPrPsInfo::Populate(const lldb_private::ProcessInstanceInfo &info,
+                           lldb::StateType process_state) {
+  ELFLinuxPrPsInfo prpsinfo{};
+  prpsinfo.pr_pid = info.GetProcessID();
+  prpsinfo.pr_nice = info.GetPriorityValue().value_or(0);
+  prpsinfo.pr_zomb = 0;
+  if (auto zombie_opt = info.IsZombie(); zombie_opt.value_or(false)) {
+    prpsinfo.pr_zomb = 1;
+  }
+  /**
+   * In the linux kernel this comes from:
+   * state = READ_ONCE(p->__state);
+   * i = state ? ffz(~state) + 1 : 0;
+   * psinfo->pr_sname = (i > 5) ? '.' : "RSDTZW"[i];
+   *
+   * So we replicate that here. From proc_pid_stats(5)
+   * R = Running
+   * S = Sleeping on uninterrutible wait
+   * D = Waiting on uninterruptable disk sleep
+   * T = Tracing stop
+   * Z = Zombie
+   * W = Paging
+   */
+  switch (process_state) {
+  case lldb::StateType::eStateSuspended:
+    prpsinfo.pr_sname = 'S';
+    prpsinfo.pr_state = 1;
+    break;
+  case lldb::StateType::eStateStopped:
+    [[fallthrough]];
+  case lldb::StateType::eStateStepping:
+    prpsinfo.pr_sname = 'T';
+    prpsinfo.pr_state = 3;
+    break;
+  case lldb::StateType::eStateUnloaded:
+    [[fallthrough]];
+  case lldb::StateType::eStateRunning:
+    prpsinfo.pr_sname = 'R';
+    prpsinfo.pr_state = 0;
+    break;
+  default:
+    break;
+  }
+
+  /**
+   * pr_flags is left as 0. The values (in linux) are specific
+   * to the kernel. We recover them from the proc filesystem
+   * but don't put them in ProcessInfo because it would really
+   * become very linux specific and the utility here seems pretty
+   * dubious
+   */
+
+  if (info.EffectiveUserIDIsValid())
+    prpsinfo.pr_uid = info.GetUserID();
+
+  if (info.EffectiveGroupIDIsValid())
+    prpsinfo.pr_gid = info.GetGroupID();
+
+  if (info.ParentProcessIDIsValid())
+    prpsinfo.pr_ppid = info.GetParentProcessID();
+
+  if (info.ProcessGroupIDIsValid())
+    prpsinfo.pr_pgrp = info.GetProcessGroupID();
+
+  if (info.ProcessSessionIDIsValid())
+    prpsinfo.pr_sid = info.GetProcessSessionID();
+
+  constexpr size_t fname_len = std::extent_v<decltype(prpsinfo.pr_fname)>;
+  static_assert(fname_len > 0, "This should always be non zero");
+  const llvm::StringRef fname = info.GetNameAsStringRef();
+  auto fname_begin = fname.begin();
+  std::copy_n(fname_begin, std::min(fname_len, fname.size()),
+              prpsinfo.pr_fname);
+  prpsinfo.pr_fname[fname_len - 1] = '\0';
+  auto args = info.GetArguments();
+  auto argentry_iterator = std::begin(args);
+  char *psargs = prpsinfo.pr_psargs;
+  char *psargs_end = std::end(prpsinfo.pr_psargs);
+  while (psargs < psargs_end && argentry_iterator != args.end()) {
+    llvm::StringRef argentry = argentry_iterator->ref();
+    size_t len =
+        std::min<size_t>(std::distance(psargs, psargs_end), argentry.size());
+    auto arg_iterator = std::begin(argentry);
+    psargs = std::copy_n(arg_iterator, len, psargs);
+    if (psargs != psargs_end)
+      *(psargs++) = ' ';
+    ++argentry_iterator;
+  }
+  *(psargs - 1) = '\0';
+  return prpsinfo;
+}
+
 // Parse SIGINFO from NOTE entry
 ELFLinuxSigInfo::ELFLinuxSigInfo() { memset(this, 0, sizeof(ELFLinuxSigInfo)); }
 
diff --git a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.h b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.h
index 2f3ed2a017790a0..3fa0b8b0eedb0b7 100644
--- a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.h
+++ b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.h
@@ -13,6 +13,7 @@
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "llvm/ADT/DenseMap.h"
+#include <optional>
 #include <string>
 
 struct compat_timeval {
@@ -20,6 +21,10 @@ struct compat_timeval {
   alignas(8) uint64_t tv_usec;
 };
 
+namespace lldb_private {
+class ProcessInstanceInfo;
+}
+
 // PRSTATUS structure's size differs based on architecture.
 // This is the layout in the x86-64 arch.
 // In the i386 case we parse it manually and fill it again
@@ -56,6 +61,9 @@ struct ELFLinuxPrStatus {
   lldb_private::Status Parse(const lldb_private::DataExtractor &data,
                              const lldb_private::ArchSpec &arch);
 
+  static std::optional<ELFLinuxPrStatus>
+  Populate(const lldb::ThreadSP &thread_sp);
+
   // Return the bytesize of the structure
   // 64 bit - just sizeof
   // 32 bit - hardcoded because we are reusing the struct, but some of the
@@ -112,6 +120,13 @@ struct ELFLinuxPrPsInfo {
   lldb_private::Status Parse(const lldb_private::DataExtractor &data,
                              const lldb_private::ArchSpec &arch);
 
+  static std::optional<ELFLinuxPrPsInfo>
+  Populate(const lldb::ProcessSP &process_sp);
+
+  static std::optional<ELFLinuxPrPsInfo>
+  Populate(const lldb_private::ProcessInstanceInfo &info,
+           lldb::StateType state);
+
   // Return the bytesize of the structure
   // 64 bit - just sizeof
   // 32 bit - hardcoded because we are reusing the struct, but some of the
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h b/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
index c99372fa110cd22..378b9fa2a598651 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
@@ -13,7 +13,7 @@
 // This declaration works around a clang module build failure.
 // It should be deleted ASAP.
 #include "llvm/Support/Error.h"
-static llvm::Expected<bool> *g_fcxx_modules_workaround;
+static llvm::Expected<bool> *g_fcxx_modules_workaround [[maybe_unused]];
 // END
 
 #include "lldb/Host/Config.h"
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index 66a762bf9b68542..81f937762e35a65 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -97,12 +97,14 @@ void DWARFUnit::ExtractUnitDIEIfNeeded() {
         *m_dwo_id, m_first_die.GetOffset()));
     return; // Can't fetch the compile unit from the dwo file.
   }
-  // If the skeleton compile unit gets its unit DIE parsed first, then this
-  // will fill in the DWO file's back pointer to this skeleton compile unit.
-  // If the DWO files get parsed on their own first the skeleton back link
-  // can be done manually in DWARFUnit::GetSkeletonCompileUnit() which will
-  // do a reverse lookup and cache the result.
-  dwo_cu->SetSkeletonUnit(this);
+
+  // Link the DWO unit to this object, if it hasn't been linked already (this
+  // can happen when we have an index, and the DWO unit is parsed first).
+  if (!dwo_cu->LinkToSkeletonUnit(*this)) {
+    SetDwoError(Status::createWithFormat(
+        "multiple compile units with Dwo ID {0:x16}", *m_dwo_id));
+    return;
+  }
 
   DWARFBaseDIE dwo_cu_die = dwo_cu->GetUnitDIEOnly();
   if (!dwo_cu_die.IsValid()) {
@@ -708,23 +710,28 @@ uint8_t DWARFUnit::GetAddressByteSize(const DWARFUnit *cu) {
 uint8_t DWARFUnit::GetDefaultAddressSize() { return 4; }
 
 DWARFCompileUnit *DWARFUnit::GetSkeletonUnit() {
-  if (m_skeleton_unit == nullptr && IsDWOUnit()) {
+  if (m_skeleton_unit.load() == nullptr && IsDWOUnit()) {
     SymbolFileDWARFDwo *dwo =
         llvm::dyn_cast_or_null<SymbolFileDWARFDwo>(&GetSymbolFileDWARF());
     // Do a reverse lookup if the skeleton compile unit wasn't set.
-    if (dwo)
-      m_skeleton_unit = dwo->GetBaseSymbolFile().GetSkeletonUnit(this);
+    DWARFUnit *candidate_skeleton_unit =
+        dwo ? dwo->GetBaseSymbolFile().GetSkeletonUnit(this) : nullptr;
+    if (candidate_skeleton_unit)
+      (void)LinkToSkeletonUnit(*candidate_skeleton_unit);
+    // Linking may fail due to a race, so be sure to return the actual value.
   }
-  return llvm::dyn_cast_or_null<DWARFCompileUnit>(m_skeleton_unit);
+  return llvm::dyn_cast_or_null<DWARFCompileUnit>(m_skeleton_unit.load());
 }
 
-void DWARFUnit::SetSkeletonUnit(DWARFUnit *skeleton_unit) {
-  // If someone is re-setting the skeleton compile unit backlink, make sure
-  // it is setting it to a valid value when it wasn't valid, or if the
-  // value in m_skeleton_unit was valid, it should be the same value.
-  assert(skeleton_unit);
-  assert(m_skeleton_unit == nullptr || m_skeleton_unit == skeleton_unit);
-  m_skeleton_unit = skeleton_unit;
+bool DWARFUnit::LinkToSkeletonUnit(DWARFUnit &skeleton_unit) {
+  DWARFUnit *expected_unit = nullptr;
+  if (m_skeleton_unit.compare_exchange_strong(expected_unit, &skeleton_unit))
+    return true;
+  if (expected_unit == &skeleton_unit) {
+    // Exchange failed because it already contained the right  value.
+    return true;
+  }
+  return false; // Already linked to a different unit.
 }
 
 bool DWARFUnit::Supports_DW_AT_APPLE_objc_complete_type() {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
index 85c37971ced8e0f..148932d67b908c2 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
@@ -170,7 +170,7 @@ class DWARFUnit : public UserID {
   /// both cases correctly and avoids crashes.
   DWARFCompileUnit *GetSkeletonUnit();
 
-  void SetSkeletonUnit(DWARFUnit *skeleton_unit);
+  bool LinkToSkeletonUnit(DWARFUnit &skeleton_unit);
 
   bool Supports_DW_AT_APPLE_objc_complete_type();
 
@@ -308,7 +308,7 @@ class DWARFUnit : public UserID {
   const llvm::DWARFAbbreviationDeclarationSet *m_abbrevs = nullptr;
   lldb_private::CompileUnit *m_lldb_cu = nullptr;
   // If this is a DWO file, we have a backlink to our skeleton compile unit.
-  DWARFUnit *m_skeleton_unit = nullptr;
+  std::atomic<DWARFUnit *> m_skeleton_unit = nullptr;
   // The compile unit debug information entry item
   DWARFDebugInfoEntry::collection m_die_array;
   mutable llvm::sys::RWMutex m_die_array_mutex;
diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp
index ddeacf18e855ee3..db8f8ce6bcbc923 100644
--- a/lldb/source/Symbol/CompileUnit.cpp
+++ b/lldb/source/Symbol/CompileUnit.cpp
@@ -213,11 +213,12 @@ VariableListSP CompileUnit::GetVariableList(bool can_create) {
   return m_variables;
 }
 
-std::vector<uint32_t> FindFileIndexes(const SupportFileList &files,
-                                      const FileSpec &file) {
+std::vector<uint32_t>
+FindFileIndexes(const SupportFileList &files, const FileSpec &file,
+                RealpathPrefixes *realpath_prefixes = nullptr) {
   std::vector<uint32_t> result;
   uint32_t idx = -1;
-  while ((idx = files.FindCompatibleIndex(idx + 1, file)) !=
+  while ((idx = files.FindCompatibleIndex(idx + 1, file, realpath_prefixes)) !=
          UINT32_MAX)
     result.push_back(idx);
   return result;
@@ -247,7 +248,8 @@ uint32_t CompileUnit::FindLineEntry(uint32_t start_idx, uint32_t line,
 
 void CompileUnit::ResolveSymbolContext(
     const SourceLocationSpec &src_location_spec,
-    SymbolContextItem resolve_scope, SymbolContextList &sc_list) {
+    SymbolContextItem resolve_scope, SymbolContextList &sc_list,
+    RealpathPrefixes *realpath_prefixes) {
   const FileSpec file_spec = src_location_spec.GetFileSpec();
   const uint32_t line = src_location_spec.GetLine().value_or(0);
   const bool check_inlines = src_location_spec.GetCheckInlines();
@@ -275,8 +277,8 @@ void CompileUnit::ResolveSymbolContext(
     return;
   }
 
-  std::vector<uint32_t> file_indexes = FindFileIndexes(GetSupportFiles(),
-                                                       file_spec);
+  std::vector<uint32_t> file_indexes =
+      FindFileIndexes(GetSupportFiles(), file_spec, realpath_prefixes);
   const size_t num_file_indexes = file_indexes.size();
   if (num_file_indexes == 0)
     return;
diff --git a/lldb/source/Symbol/SaveCoreOptions.cpp b/lldb/source/Symbol/SaveCoreOptions.cpp
index 0f6fdac1ce22e6a..3c4ca2d852b76aa 100644
--- a/lldb/source/Symbol/SaveCoreOptions.cpp
+++ b/lldb/source/Symbol/SaveCoreOptions.cpp
@@ -8,6 +8,8 @@
 
 #include "lldb/Symbol/SaveCoreOptions.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Target/Thread.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -46,8 +48,86 @@ SaveCoreOptions::GetOutputFile() const {
   return m_file;
 }
 
+Status SaveCoreOptions::SetProcess(lldb::ProcessSP process_sp) {
+  Status error;
+  if (!process_sp) {
+    ClearProcessSpecificData();
+    m_process_sp.reset();
+    return error;
+  }
+
+  if (!process_sp->IsValid()) {
+    error.SetErrorString("Cannot assign an invalid process.");
+    return error;
+  }
+
+  // Don't clear any process specific data if the process is the same.
+  if (m_process_sp == process_sp)
+    return error;
+
+  ClearProcessSpecificData();
+  m_process_sp = process_sp;
+  return error;
+}
+
+Status SaveCoreOptions::AddThread(lldb::ThreadSP thread_sp) {
+  Status error;
+  if (!thread_sp) {
+    error.SetErrorString("invalid thread");
+    return error;
+  }
+
+  if (m_process_sp) {
+    if (m_process_sp != thread_sp->GetProcess()) {
+      error.SetErrorString("Cannot add a thread from a different process.");
+      return error;
+    }
+  } else {
+    m_process_sp = thread_sp->GetProcess();
+  }
+
+  m_threads_to_save.insert(thread_sp->GetID());
+  return error;
+}
+
+bool SaveCoreOptions::RemoveThread(lldb::ThreadSP thread_sp) {
+  return thread_sp && m_threads_to_save.erase(thread_sp->GetID()) > 0;
+}
+
+bool SaveCoreOptions::ShouldThreadBeSaved(lldb::tid_t tid) const {
+  // If the user specified no threads to save, then we save all threads.
+  if (m_threads_to_save.empty())
+    return true;
+  return m_threads_to_save.count(tid) > 0;
+}
+
+Status SaveCoreOptions::EnsureValidConfiguration(
+    lldb::ProcessSP process_sp) const {
+  Status error;
+  std::string error_str;
+  if (!m_threads_to_save.empty() && GetStyle() == lldb::eSaveCoreFull)
+    error_str += "Cannot save a full core with a subset of threads\n";
+
+  if (m_process_sp && m_process_sp != process_sp)
+    error_str += "Cannot save core for process using supplied core options. "
+                 "Options were constructed targeting a different process. \n";
+
+  if (!error_str.empty())
+    error.SetErrorString(error_str);
+
+  return error;
+}
+
+void SaveCoreOptions::ClearProcessSpecificData() { 
+  // Deliberately not following the formatter style here to indicate that
+  // this method will be expanded in the future.
+  m_threads_to_save.clear(); 
+}
+
 void SaveCoreOptions::Clear() {
   m_file = std::nullopt;
   m_plugin_name = std::nullopt;
   m_style = std::nullopt;
+  m_threads_to_save.clear();
+  m_process_sp.reset();
 }
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index e3c4f2ee398cc46..3c9247fdbbbc968 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -6548,8 +6548,9 @@ static void AddRegion(const MemoryRegionInfo &region, bool try_dirty_pages,
 }
 
 static void SaveOffRegionsWithStackPointers(
-    Process &process, const MemoryRegionInfos &regions,
-    Process::CoreFileMemoryRanges &ranges, std::set<addr_t> &stack_ends) {
+    Process &process, const SaveCoreOptions &core_options,
+    const MemoryRegionInfos &regions, Process::CoreFileMemoryRanges &ranges,
+    std::set<addr_t> &stack_ends) {
   const bool try_dirty_pages = true;
 
   // Before we take any dump, we want to save off the used portions of the
@@ -6571,10 +6572,16 @@ static void SaveOffRegionsWithStackPointers(
     if (process.GetMemoryRegionInfo(sp, sp_region).Success()) {
       const size_t stack_head = (sp - red_zone);
       const size_t stack_size = sp_region.GetRange().GetRangeEnd() - stack_head;
+      // Even if the SaveCoreOption doesn't want us to save the stack
+      // we still need to populate the stack_ends set so it doesn't get saved
+      // off in other calls
       sp_region.GetRange().SetRangeBase(stack_head);
       sp_region.GetRange().SetByteSize(stack_size);
       stack_ends.insert(sp_region.GetRange().GetRangeEnd());
-      AddRegion(sp_region, try_dirty_pages, ranges);
+      // This will return true if the threadlist the user specified is empty,
+      // or contains the thread id from thread_sp.
+      if (core_options.ShouldThreadBeSaved(thread_sp->GetID()))
+        AddRegion(sp_region, try_dirty_pages, ranges);
     }
   }
 }
@@ -6643,10 +6650,11 @@ static void GetCoreFileSaveRangesStackOnly(
   }
 }
 
-Status Process::CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
+Status Process::CalculateCoreFileSaveRanges(const SaveCoreOptions &options,
                                             CoreFileMemoryRanges &ranges) {
   lldb_private::MemoryRegionInfos regions;
   Status err = GetMemoryRegions(regions);
+  SaveCoreStyle core_style = options.GetStyle();
   if (err.Fail())
     return err;
   if (regions.empty())
@@ -6656,7 +6664,7 @@ Status Process::CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
                   "eSaveCoreUnspecified");
 
   std::set<addr_t> stack_ends;
-  SaveOffRegionsWithStackPointers(*this, regions, ranges, stack_ends);
+  SaveOffRegionsWithStackPointers(*this, options, regions, ranges, stack_ends);
 
   switch (core_style) {
   case eSaveCoreUnspecified:
@@ -6684,6 +6692,18 @@ Status Process::CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
   return Status(); // Success!
 }
 
+std::vector<ThreadSP>
+Process::CalculateCoreFileThreadList(const SaveCoreOptions &core_options) {
+  std::vector<ThreadSP> thread_list;
+  for (const lldb::ThreadSP &thread_sp : m_thread_list.Threads()) {
+    if (core_options.ShouldThreadBeSaved(thread_sp->GetID())) {
+      thread_list.push_back(thread_sp);
+    }
+  }
+
+  return thread_list;
+}
+
 void Process::SetAddressableBitMasks(AddressableBits bit_masks) {
   uint32_t low_memory_addr_bits = bit_masks.GetLowmemAddressableBits();
   uint32_t high_memory_addr_bits = bit_masks.GetHighmemAddressableBits();
diff --git a/lldb/source/Target/Statistics.cpp b/lldb/source/Target/Statistics.cpp
index 583d1524881fc38..390e04cebf6be6c 100644
--- a/lldb/source/Target/Statistics.cpp
+++ b/lldb/source/Target/Statistics.cpp
@@ -192,6 +192,10 @@ TargetStats::ToJSON(Target &target,
   }
   target_metrics_json.try_emplace("sourceMapDeduceCount",
                                   m_source_map_deduce_count);
+  target_metrics_json.try_emplace("sourceRealpathAttemptCount",
+                                  m_source_realpath_attempt_count);
+  target_metrics_json.try_emplace("sourceRealpathCompatibleCount",
+                                  m_source_realpath_compatible_count);
   return target_metrics_json;
 }
 
@@ -220,6 +224,14 @@ void TargetStats::IncreaseSourceMapDeduceCount() {
   ++m_source_map_deduce_count;
 }
 
+void TargetStats::IncreaseSourceRealpathAttemptCount(uint32_t count) {
+  m_source_realpath_attempt_count += count;
+}
+
+void TargetStats::IncreaseSourceRealpathCompatibleCount(uint32_t count) {
+  m_source_realpath_compatible_count += count;
+}
+
 bool DebuggerStats::g_collecting_stats = false;
 
 llvm::json::Value DebuggerStats::ReportStatistics(
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 129683c43f0c1a0..5a5d689e03fbc04 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -60,6 +60,7 @@
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/Timer.h"
@@ -4354,6 +4355,13 @@ InlineStrategy TargetProperties::GetInlineStrategy() const {
       static_cast<InlineStrategy>(g_target_properties[idx].default_uint_value));
 }
 
+// Returning RealpathPrefixes, but the setting's type is FileSpecList. We do
+// this because we want the FileSpecList to normalize the file paths for us.
+RealpathPrefixes TargetProperties::GetSourceRealpathPrefixes() const {
+  const uint32_t idx = ePropertySourceRealpathPrefixes;
+  return RealpathPrefixes(GetPropertyAtIndexAs<FileSpecList>(idx, {}));
+}
+
 llvm::StringRef TargetProperties::GetArg0() const {
   const uint32_t idx = ePropertyArg0;
   return GetPropertyAtIndexAs<llvm::StringRef>(
diff --git a/lldb/source/Target/TargetProperties.td b/lldb/source/Target/TargetProperties.td
index ef538678670fea0..421252aa4aea26b 100644
--- a/lldb/source/Target/TargetProperties.td
+++ b/lldb/source/Target/TargetProperties.td
@@ -150,6 +150,9 @@ let Definition = "target" in {
     DefaultEnumValue<"eInlineBreakpointsAlways">,
     EnumValues<"OptionEnumValues(g_inline_breakpoint_enums)">,
     Desc<"The strategy to use when settings breakpoints by file and line. Breakpoint locations can end up being inlined by the compiler, so that a compile unit 'a.c' might contain an inlined function from another source file. Usually this is limited to breakpoint locations from inlined functions from header or other include files, or more accurately non-implementation source files. Sometimes code might #include implementation files and cause inlined breakpoint locations in inlined implementation files. Always checking for inlined breakpoint locations can be expensive (memory and time), so if you have a project with many headers and find that setting breakpoints is slow, then you can change this setting to headers. This setting allows you to control exactly which strategy is used when setting file and line breakpoints.">;
+  def SourceRealpathPrefixes: Property<"source-realpath-prefixes", "FileSpecList">,
+    DefaultStringValue<"">,
+    Desc<"Realpath any source paths that start with one of these prefixes. If the debug info contains symlinks which match the original source file's basename but don't match its location that the user will use to set breakpoints, then this setting can help resolve breakpoints correctly. This handles both symlinked files and directories. Wild card prefixes: An empty string matches all paths. A forward slash matches absolute paths.">;
   def DisassemblyFlavor: Property<"x86-disassembly-flavor", "Enum">,
     DefaultEnumValue<"eX86DisFlavorDefault">,
     EnumValues<"OptionEnumValues(g_x86_dis_flavor_value_types)">,
diff --git a/lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp b/lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp
index 0a939d55f4ce493..806ba95c508b7ca 100644
--- a/lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp
+++ b/lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp
@@ -29,10 +29,10 @@ ThreadPlanSingleThreadTimeout::ThreadPlanSingleThreadTimeout(
     : ThreadPlan(ThreadPlan::eKindSingleThreadTimeout, "Single thread timeout",
                  thread, eVoteNo, eVoteNoOpinion),
       m_info(info), m_state(State::WaitTimeout) {
-  // TODO: reuse m_timer_thread without recreation.
-  m_timer_thread = std::thread(TimeoutThreadFunc, this);
   m_info->m_isAlive = true;
   m_state = m_info->m_last_state;
+  // TODO: reuse m_timer_thread without recreation.
+  m_timer_thread = std::thread(TimeoutThreadFunc, this);
 }
 
 ThreadPlanSingleThreadTimeout::~ThreadPlanSingleThreadTimeout() {
diff --git a/lldb/source/Utility/CMakeLists.txt b/lldb/source/Utility/CMakeLists.txt
index e9954d66cd1a528..397db0e89760234 100644
--- a/lldb/source/Utility/CMakeLists.txt
+++ b/lldb/source/Utility/CMakeLists.txt
@@ -51,6 +51,7 @@ add_lldb_library(lldbUtility NO_INTERNAL_DEPENDENCIES
   Log.cpp
   NameMatches.cpp
   ProcessInfo.cpp
+  RealpathPrefixes.cpp
   RegisterValue.cpp
   RegularExpression.cpp
   Instrumentation.cpp
diff --git a/lldb/source/Utility/FileSpecList.cpp b/lldb/source/Utility/FileSpecList.cpp
index 7647e04a8204516..5852367f77827f6 100644
--- a/lldb/source/Utility/FileSpecList.cpp
+++ b/lldb/source/Utility/FileSpecList.cpp
@@ -7,7 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Utility/FileSpecList.h"
+#include "lldb/Target/Statistics.h"
+#include "lldb/Target/Target.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/Stream.h"
 
 #include <cstdint>
@@ -108,52 +113,85 @@ size_t SupportFileList::FindFileIndex(size_t start_idx,
                          });
 }
 
-size_t SupportFileList::FindCompatibleIndex(size_t start_idx,
-                                            const FileSpec &file_spec) const {
-  const size_t num_files = m_files.size();
-  if (start_idx >= num_files)
-    return UINT32_MAX;
+enum IsCompatibleResult {
+  kNoMatch = 0,
+  kOnlyFileMatch = 1,
+  kBothDirectoryAndFileMatch = 2,
+};
 
+IsCompatibleResult IsCompatible(const FileSpec &curr_file,
+                                const FileSpec &file_spec) {
   const bool file_spec_relative = file_spec.IsRelative();
   const bool file_spec_case_sensitive = file_spec.IsCaseSensitive();
   // When looking for files, we will compare only the filename if the directory
   // argument is empty in file_spec
   const bool full = !file_spec.GetDirectory().IsEmpty();
 
+  // Always start by matching the filename first
+  if (!curr_file.FileEquals(file_spec))
+    return IsCompatibleResult::kNoMatch;
+
+  // Only compare the full name if the we were asked to and if the current
+  // file entry has a directory. If it doesn't have a directory then we only
+  // compare the filename.
+  if (FileSpec::Equal(curr_file, file_spec, full)) {
+    return IsCompatibleResult::kBothDirectoryAndFileMatch;
+  } else if (curr_file.IsRelative() || file_spec_relative) {
+    llvm::StringRef curr_file_dir = curr_file.GetDirectory().GetStringRef();
+    if (curr_file_dir.empty())
+      // Basename match only for this file in the list
+      return IsCompatibleResult::kBothDirectoryAndFileMatch;
+
+    // Check if we have a relative path in our file list, or if "file_spec" is
+    // relative, if so, check if either ends with the other.
+    llvm::StringRef file_spec_dir = file_spec.GetDirectory().GetStringRef();
+    // We have a relative path in our file list, it matches if the
+    // specified path ends with this path, but we must ensure the full
+    // component matches (we don't want "foo/bar.cpp" to match "oo/bar.cpp").
+    auto is_suffix = [](llvm::StringRef a, llvm::StringRef b,
+                        bool case_sensitive) -> bool {
+      if (case_sensitive ? a.consume_back(b) : a.consume_back_insensitive(b))
+        return a.empty() || a.ends_with("/");
+      return false;
+    };
+    const bool case_sensitive =
+        file_spec_case_sensitive || curr_file.IsCaseSensitive();
+    if (is_suffix(curr_file_dir, file_spec_dir, case_sensitive) ||
+        is_suffix(file_spec_dir, curr_file_dir, case_sensitive))
+      return IsCompatibleResult::kBothDirectoryAndFileMatch;
+  }
+  return IsCompatibleResult::kOnlyFileMatch;
+}
+
+size_t SupportFileList::FindCompatibleIndex(
+    size_t start_idx, const FileSpec &file_spec,
+    RealpathPrefixes *realpath_prefixes) const {
+  const size_t num_files = m_files.size();
+  if (start_idx >= num_files)
+    return UINT32_MAX;
+
   for (size_t idx = start_idx; idx < num_files; ++idx) {
     const FileSpec &curr_file = m_files[idx]->GetSpecOnly();
 
-    // Always start by matching the filename first
-    if (!curr_file.FileEquals(file_spec))
-      continue;
-
-    // Only compare the full name if the we were asked to and if the current
-    // file entry has the a directory. If it doesn't have a directory then we
-    // only compare the filename.
-    if (FileSpec::Equal(curr_file, file_spec, full)) {
+    IsCompatibleResult result = IsCompatible(curr_file, file_spec);
+    if (result == IsCompatibleResult::kBothDirectoryAndFileMatch)
       return idx;
-    } else if (curr_file.IsRelative() || file_spec_relative) {
-      llvm::StringRef curr_file_dir = curr_file.GetDirectory().GetStringRef();
-      if (curr_file_dir.empty())
-        return idx; // Basename match only for this file in the list
-
-      // Check if we have a relative path in our file list, or if "file_spec" is
-      // relative, if so, check if either ends with the other.
-      llvm::StringRef file_spec_dir = file_spec.GetDirectory().GetStringRef();
-      // We have a relative path in our file list, it matches if the
-      // specified path ends with this path, but we must ensure the full
-      // component matches (we don't want "foo/bar.cpp" to match "oo/bar.cpp").
-      auto is_suffix = [](llvm::StringRef a, llvm::StringRef b,
-                          bool case_sensitive) -> bool {
-        if (case_sensitive ? a.consume_back(b) : a.consume_back_insensitive(b))
-          return a.empty() || a.ends_with("/");
-        return false;
-      };
-      const bool case_sensitive =
-          file_spec_case_sensitive || curr_file.IsCaseSensitive();
-      if (is_suffix(curr_file_dir, file_spec_dir, case_sensitive) ||
-          is_suffix(file_spec_dir, curr_file_dir, case_sensitive))
-        return idx;
+
+    if (realpath_prefixes && result == IsCompatibleResult::kOnlyFileMatch) {
+      if (std::optional<FileSpec> resolved_curr_file =
+              realpath_prefixes->ResolveSymlinks(curr_file)) {
+        if (IsCompatible(*resolved_curr_file, file_spec) ==
+            IsCompatibleResult::kBothDirectoryAndFileMatch) {
+          // Stats and logging.
+          realpath_prefixes->IncreaseSourceRealpathCompatibleCount();
+          Log *log = GetLog(LLDBLog::Source);
+          LLDB_LOGF(log,
+                    "Realpath'ed support file %s is compatible to input file",
+                    resolved_curr_file->GetPath().c_str());
+          // We found a match
+          return idx;
+        }
+      }
     }
   }
 
diff --git a/lldb/source/Utility/RealpathPrefixes.cpp b/lldb/source/Utility/RealpathPrefixes.cpp
new file mode 100644
index 000000000000000..f6e1d17e30c6960
--- /dev/null
+++ b/lldb/source/Utility/RealpathPrefixes.cpp
@@ -0,0 +1,71 @@
+//===-- RealpathPrefixes.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Utility/RealpathPrefixes.h"
+
+#include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+#include "lldb/lldb-private-types.h"
+
+using namespace lldb_private;
+
+RealpathPrefixes::RealpathPrefixes(
+    const FileSpecList &file_spec_list,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs)
+    : m_fs(fs) {
+  m_prefixes.reserve(file_spec_list.GetSize());
+  for (const FileSpec &file_spec : file_spec_list) {
+    m_prefixes.emplace_back(file_spec.GetPath());
+  }
+}
+
+std::optional<FileSpec>
+RealpathPrefixes::ResolveSymlinks(const FileSpec &file_spec) {
+  if (m_prefixes.empty())
+    return std::nullopt;
+
+  // Test if `b` is a *path* prefix of `a` (not just *string* prefix).
+  // E.g. "/foo/bar" is a path prefix of "/foo/bar/baz" but not "/foo/barbaz".
+  auto is_path_prefix = [](llvm::StringRef a, llvm::StringRef b,
+                           bool case_sensitive,
+                           llvm::sys::path::Style style) -> bool {
+    if (case_sensitive ? a.consume_front(b) : a.consume_front_insensitive(b))
+      // If `b` isn't "/", then it won't end with "/" because it comes from
+      // `FileSpec`. After `a` consumes `b`, `a` should either be empty (i.e.
+      // `a` == `b`) or end with "/" (the remainder of `a` is a subdirectory).
+      return b == "/" || a.empty() ||
+             llvm::sys::path::is_separator(a[0], style);
+    return false;
+  };
+  std::string file_spec_path = file_spec.GetPath();
+  for (const std::string &prefix : m_prefixes) {
+    if (is_path_prefix(file_spec_path, prefix, file_spec.IsCaseSensitive(),
+                       file_spec.GetPathStyle())) {
+      // Stats and logging.
+      IncreaseSourceRealpathAttemptCount();
+      Log *log = GetLog(LLDBLog::Source);
+      LLDB_LOGF(log, "Realpath'ing support file %s", file_spec_path.c_str());
+
+      // One prefix matched. Try to realpath.
+      PathSmallString buff;
+      std::error_code ec = m_fs->getRealPath(file_spec_path, buff);
+      if (ec)
+        return std::nullopt;
+      FileSpec realpath(buff, file_spec.GetPathStyle());
+
+      // Only return realpath if it is different from the original file_spec.
+      if (realpath != file_spec)
+        return realpath;
+      return std::nullopt;
+    }
+  }
+  // No prefix matched
+  return std::nullopt;
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/Makefile
new file mode 100644
index 000000000000000..10495940055b63d
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/TestBreakpoint.py b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/TestBreakpoint.py
new file mode 100644
index 000000000000000..5dc2af73f3647d4
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/TestBreakpoint.py
@@ -0,0 +1,158 @@
+"""
+Test lldb breakpoint with symlinks/realpath and source-map.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil, lldbplatformutil
+
+
+class BreakpointTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def setUp(self):
+        # Call super's setUp().
+        TestBase.setUp(self)
+        # Find the line number to break inside main().
+        self.line_in_main = line_number("main.c", "// Set break point at this line.")
+        self.line_in_foo = line_number("real/foo.h", "// Set break point at this line.")
+        self.line_in_bar = line_number("real/bar.h", "// Set break point at this line.")
+        self.line_in_qux = line_number("real/qux.h", "// Set break point at this line.")
+        # disable "There is a running process, kill it and restart?" prompt
+        self.runCmd("settings set auto-confirm true")
+        self.addTearDownHook(lambda: self.runCmd("settings clear auto-confirm"))
+
+    def buildAndCreateTarget(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        # Create a target by the debugger.
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+    @skipIf(oslist=["windows"])
+    def test_file_line_breakpoint_realpath_and_source_map(self):
+        """Test file/line breakpoint with realpathing and source-mapping."""
+        self.buildAndCreateTarget()
+        cwd = os.getcwd()
+
+        ######################################################################
+        # Baseline
+        # --------------------------------------------------------------------
+        # Breakpoints should be resolved with paths which are in the line-table.
+        lldbutil.run_break_set_by_file_and_line(
+            self, "main.c", self.line_in_main, num_expected_locations=1, loc_exact=True
+        )
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "symlink1/foo.h",
+            self.line_in_foo,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "symlink2/bar.h",
+            self.line_in_bar,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "symlink2/qux.h",
+            self.line_in_qux,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+
+        ######################################################################
+        # Symlinked file
+        # --------------------------------------------------------------------
+        # - `symlink1/foo.h` is a symlink file, pointing at `real/foo.h`
+        # - main.c includes `symlink1/foo.h`.
+        # - As a result, the line-table contains a support file `(test_base_dir)/symlink1/foo.h`
+        # - Setting a breakpoint for `real/foo.h` won't be resolved, because it doesn't match the above path.
+        # - Setting a realpath prefix to the current working directory will cause the above support file to be realpath'ed to `(test_base_dir)/real/foo.h`
+        # - Now setting a breakpoint for `real/foo.h` will be resolved.
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/foo.h",
+            self.line_in_foo,
+            num_expected_locations=0,
+            loc_exact=True,
+        )
+        self.runCmd(f'settings set target.source-realpath-prefixes "{cwd}"')
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/foo.h",
+            self.line_in_foo,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        # Clear settings so that the test below won't be affected.
+        self.runCmd("settings clear target.source-realpath-prefixes")
+
+        ######################################################################
+        # Symlinked directory
+        # --------------------------------------------------------------------
+        # - `symlink2` is a symlink directory, pointing at `real`.
+        # - So, `symlink2/bar.h` will be realpath'ed to `real/bar.h`.
+        # - main.c includes `symlink2/bar.h`.
+        # - As a result, the line-table contains a support file `(test_base_dir)/symlink2/bar.h`
+        # - Setting a breakpoint for `real/bar.h` won't be resolved, because it doesn't match the above path.
+        # - Setting a realpath prefix to the current working directory will cause the above support file to be realpath'ed to `(test_base_dir)/real/bar.h`
+        # - Now setting a breakpoint for `real/bar.h` will be resolved.
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/bar.h",
+            self.line_in_foo,
+            num_expected_locations=0,
+            loc_exact=True,
+        )
+        self.runCmd(f'settings set target.source-realpath-prefixes "{cwd}"')
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/bar.h",
+            self.line_in_foo,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        # Clear settings so that the test below won't be affected.
+        self.runCmd("settings clear target.source-realpath-prefixes")
+
+        ######################################################################
+        # Symlink + source-map
+        # --------------------------------------------------------------------
+        # - `symlink2` is a symlink directory, pointing at `real`.
+        # - So, `symlink2/qux.h` will be realpath'ed to `real/qux.h`.
+        # - main.c includes `symlink2/qux.h`.
+        # - As a result, the line-table contains a support file `(test_base_dir)/symlink2/qux.h`
+        # - Setting a realpath prefix to the current working directory will cause the above support file to be realpath'ed to `(test_base_dir)/real/qux.h`
+        # - Setting a breakpoint for `to-be-mapped/qux.h` won't be resolved, because it doesn't match the above path.
+        # - After setting a source-map, setting the same breakpoint will be resolved, because the input path `to-be-mapped/qux.h` is reverse-mapped to `real/qux.h`, which matches the realpath'ed support file.
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/qux.h",
+            self.line_in_foo,
+            num_expected_locations=0,
+            loc_exact=True,
+        )
+        self.runCmd(f'settings set target.source-realpath-prefixes "{cwd}"')
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "to-be-mapped/qux.h",
+            self.line_in_foo,
+            num_expected_locations=0,
+            loc_exact=True,
+        )
+        self.runCmd('settings set target.source-map "real" "to-be-mapped"')
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "to-be-mapped/qux.h",
+            self.line_in_foo,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        # Clear settings so that the test below won't be affected.
+        self.runCmd("settings clear target.source-realpath-prefixes")
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/main.c
new file mode 100644
index 000000000000000..716e7a91715f9b2
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/main.c
@@ -0,0 +1,10 @@
+#include "symlink1/foo.h"
+#include "symlink2/bar.h"
+#include "symlink2/qux.h"
+
+int main(int argc, char const *argv[]) {
+  int a = foo();    // 1
+  int b = bar();    // 2
+  int c = qux();    // 3
+  return a + b - c; // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/bar.h b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/bar.h
new file mode 100644
index 000000000000000..40155eb2075ab65
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/bar.h
@@ -0,0 +1,3 @@
+int bar() {
+  return 2; // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/foo.h b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/foo.h
new file mode 100644
index 000000000000000..a4ceefecf012389
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/foo.h
@@ -0,0 +1,3 @@
+int foo() {
+  return 1; // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/qux.h b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/qux.h
new file mode 100644
index 000000000000000..a90e0feba30aa55
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/qux.h
@@ -0,0 +1,3 @@
+int qux() {
+  return 3; // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink1/foo.h b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink1/foo.h
new file mode 120000
index 000000000000000..c9001b4753bba67
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink1/foo.h
@@ -0,0 +1 @@
+../real/foo.h
\ No newline at end of file
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink2 b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink2
new file mode 120000
index 000000000000000..ac558a3e1bf4442
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink2
@@ -0,0 +1 @@
+real
\ No newline at end of file
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/to-be-mapped/README.md b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/to-be-mapped/README.md
new file mode 100644
index 000000000000000..5067af8eed047aa
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/to-be-mapped/README.md
@@ -0,0 +1 @@
+This is an empty folder just so that `settings set target.source-map "real" "to-be-mapped"` can be run successfully - it requires tha the latter path is valid.
diff --git a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
index 96511d790271fe2..5abaa05a90f63e6 100644
--- a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
+++ b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
@@ -199,3 +199,104 @@ def test_save_linux_mini_dump(self):
                 os.unlink(core_sb_dirty)
             if os.path.isfile(core_sb_full):
                 os.unlink(core_sb_full)
+
+    @skipUnlessArch("x86_64")
+    @skipUnlessPlatform(["linux"])
+    def test_save_linux_mini_dump_thread_options(self):
+        """Test that we can save a Linux mini dump
+        with a subset of threads"""
+
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        thread_subset_dmp = self.getBuildArtifact("core.thread.subset.dmp")
+        try:
+            target = self.dbg.CreateTarget(exe)
+            process = target.LaunchSimple(
+                None, None, self.get_process_working_directory()
+            )
+            self.assertState(process.GetState(), lldb.eStateStopped)
+
+            thread_to_include = process.GetThreadAtIndex(0)
+            options = lldb.SBSaveCoreOptions()
+            thread_subset_spec = lldb.SBFileSpec(thread_subset_dmp)
+            options.AddThread(thread_to_include)
+            options.SetOutputFile(thread_subset_spec)
+            options.SetPluginName("minidump")
+            options.SetStyle(lldb.eSaveCoreStackOnly)
+            error = process.SaveCore(options)
+            self.assertTrue(error.Success())
+
+            core_target = self.dbg.CreateTarget(None)
+            core_process = core_target.LoadCore(thread_subset_dmp)
+
+            self.assertTrue(core_process, PROCESS_IS_VALID)
+            self.assertEqual(core_process.GetNumThreads(), 1)
+            saved_thread = core_process.GetThreadAtIndex(0)
+            expected_thread = process.GetThreadAtIndex(0)
+            self.assertEqual(expected_thread.GetThreadID(), saved_thread.GetThreadID())
+            expected_sp = expected_thread.GetFrameAtIndex(0).GetSP()
+            saved_sp = saved_thread.GetFrameAtIndex(0).GetSP()
+            self.assertEqual(expected_sp, saved_sp)
+            expected_region = lldb.SBMemoryRegionInfo()
+            saved_region = lldb.SBMemoryRegionInfo()
+            error = core_process.GetMemoryRegionInfo(saved_sp, saved_region)
+            self.assertTrue(error.Success(), error.GetCString())
+            error = process.GetMemoryRegionInfo(expected_sp, expected_region)
+            self.assertTrue(error.Success(), error.GetCString())
+            self.assertEqual(
+                expected_region.GetRegionBase(), saved_region.GetRegionBase()
+            )
+            self.assertEqual(
+                expected_region.GetRegionEnd(), saved_region.GetRegionEnd()
+            )
+
+        finally:
+            self.assertTrue(self.dbg.DeleteTarget(target))
+            if os.path.isfile(thread_subset_dmp):
+                os.unlink(thread_subset_dmp)
+
+    @skipUnlessArch("x86_64")
+    @skipUnlessPlatform(["linux"])
+    def test_save_linux_mini_dump_default_options(self):
+        """Test that we can save a Linux mini dump with default SBSaveCoreOptions"""
+
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        default_value_file = self.getBuildArtifact("core.defaults.dmp")
+        try:
+            target = self.dbg.CreateTarget(exe)
+            process = target.LaunchSimple(
+                None, None, self.get_process_working_directory()
+            )
+            self.assertState(process.GetState(), lldb.eStateStopped)
+
+            process_info = process.GetProcessInfo()
+            expected_pid = process_info.GetProcessID() if process_info.IsValid() else -1
+            expected_modules = target.modules
+            expected_threads = []
+            stacks_to_sp_map = {}
+            expected_pid = process.GetProcessInfo().GetProcessID()
+
+            for thread_idx in range(process.GetNumThreads()):
+                thread = process.GetThreadAtIndex(thread_idx)
+                thread_id = thread.GetThreadID()
+                expected_threads.append(thread_id)
+                stacks_to_sp_map[thread_id] = thread.GetFrameAtIndex(0).GetSP()
+
+
+            # This is almost identical to the single thread test case because
+            # minidump defaults to stacks only, so we want to see if the
+            # default options work as expected.
+            options = lldb.SBSaveCoreOptions()
+            default_value_spec = lldb.SBFileSpec(default_value_file)
+            options.SetOutputFile(default_value_spec)
+            options.SetPluginName("minidump")
+            error = process.SaveCore(options)
+            self.assertTrue(error.Success())
+
+            self.verify_core_file(default_value_file, expected_pid, expected_modules, expected_threads, stacks_to_sp_map)
+
+        finally:
+            self.assertTrue(self.dbg.DeleteTarget(target))
+            if os.path.isfile(default_value_file):
+                os.unlink(default_value_file)
diff --git a/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py b/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py
index c509e81d8951a8f..40d0cc7e96ff489 100644
--- a/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py
+++ b/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py
@@ -4,8 +4,26 @@
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 
-
 class SBSaveCoreOptionsAPICase(TestBase):
+    basic_minidump = "basic_minidump.yaml"
+    basic_minidump_different_pid = "basic_minidump_different_pid.yaml"
+
+    def get_process_from_yaml(self, yaml_file):
+        minidump_path = self.getBuildArtifact(os.path.basename(yaml_file) + ".dmp")
+        print ("minidump_path: " + minidump_path)
+        self.yaml2obj(yaml_file, minidump_path)
+        self.assertTrue(os.path.exists(minidump_path), "yaml2obj did not emit a minidump file")
+        target = self.dbg.CreateTarget(None)
+        process = target.LoadCore(minidump_path)
+        self.assertTrue(process.IsValid(), "Process is not valid")
+        return process
+
+    def get_basic_process(self):
+        return self.get_process_from_yaml(self.basic_minidump)
+
+    def get_basic_process_different_pid(self):
+        return self.get_process_from_yaml(self.basic_minidump_different_pid)
+
     def test_plugin_name_assignment(self):
         """Test assignment ensuring valid plugin names only."""
         options = lldb.SBSaveCoreOptions()
@@ -26,3 +44,38 @@ def test_default_corestyle_behavior(self):
         """Test that the default core style is unspecified."""
         options = lldb.SBSaveCoreOptions()
         self.assertEqual(options.GetStyle(), lldb.eSaveCoreUnspecified)
+
+    def test_adding_and_removing_thread(self):
+        """Test adding and removing a thread from save core options."""
+        self.assertTrue(self.dbg)
+        options = lldb.SBSaveCoreOptions()
+        process = self.get_basic_process()
+        self.assertTrue(process.IsValid(), "Process is not valid")
+        thread = process.GetThreadAtIndex(0)
+        error = options.AddThread(thread)
+        self.assertTrue(error.Success(), error.GetCString())
+        removed_success = options.RemoveThread(thread)
+        self.assertTrue(removed_success)
+        removed_success = options.RemoveThread(thread)
+        self.assertFalse(removed_success)
+
+
+    def test_adding_thread_different_process(self):
+        """Test adding and removing a thread from save core options."""
+        options = lldb.SBSaveCoreOptions()
+        process = self.get_basic_process()
+        process_2 = self.get_basic_process_different_pid()
+        thread = process.GetThreadAtIndex(0)
+        error = options.AddThread(thread)
+        self.assertTrue(error.Success())
+        thread_2 = process_2.GetThreadAtIndex(0)
+        error = options.AddThread(thread_2)
+        self.assertTrue(error.Fail())
+        options.Clear()
+        error = options.AddThread(thread_2)
+        self.assertTrue(error.Success())
+        options.SetProcess(process)
+        error = options.AddThread(thread_2)
+        self.assertTrue(error.Fail())
+        error = options.AddThread(thread)
+        self.assertTrue(error.Success())
diff --git a/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump.yaml b/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump.yaml
new file mode 100644
index 000000000000000..993c7da21225a98
--- /dev/null
+++ b/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump.yaml
@@ -0,0 +1,26 @@
+--- !minidump
+Streams:
+  - Type:            SystemInfo
+    Processor Arch:  AMD64
+    Processor Level: 6
+    Processor Revision: 15876
+    Number of Processors: 40
+    Platform ID:     Linux
+    CSD Version:     'Linux 3.13.0-91-generic #138-Ubuntu SMP Fri Jun 24 17:00:34 UTC 2016 x86_64'
+    CPU:
+      Vendor ID:       GenuineIntel
+      Version Info:    0x00000000
+      Feature Info:    0x00000000
+  - Type:            LinuxProcStatus
+    Text:             |
+      Name:	test-yaml
+      Umask:	0002
+      State:	t (tracing stop)
+      Pid:	8567
+  - Type:            ThreadList
+    Threads:
+      - Thread Id:       0x000074DD
+        Context:         0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000B0010000000000033000000000000000000000002020100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040109600000000000100000000000000000000000000000068E7D0C8FF7F000068E7D0C8FF7F000097E6D0C8FF7F000010109600000000000000000000000000020000000000000088E4D0C8FF7F0000603FFF85C77F0000F00340000000000080E7D0C8FF7F000000000000000000000000000000000000E0034000000000007F0300000000000000000000000000000000000000000000801F0000FFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF252525252525252525252525252525250000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000FF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+        Stack:
+          Start of Memory Range: 0x00007FFFC8D0E000
+          Content:               'DEADBEEF'
diff --git a/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump_different_pid.yaml b/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump_different_pid.yaml
new file mode 100644
index 000000000000000..7393f1fe62af3d1
--- /dev/null
+++ b/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump_different_pid.yaml
@@ -0,0 +1,26 @@
+--- !minidump
+Streams:
+  - Type:            SystemInfo
+    Processor Arch:  AMD64
+    Processor Level: 6
+    Processor Revision: 15876
+    Number of Processors: 40
+    Platform ID:     Linux
+    CSD Version:     'Linux 3.13.0-91-generic #138-Ubuntu SMP Fri Jun 24 17:00:34 UTC 2016 x86_64'
+    CPU:
+      Vendor ID:       GenuineIntel
+      Version Info:    0x00000000
+      Feature Info:    0x00000000
+  - Type:            LinuxProcStatus
+    Text:             |
+      Name:	test-yaml
+      Umask:	0002
+      State:	t (tracing stop)
+      Pid:	1967
+  - Type:            ThreadList
+    Threads:
+      - Thread Id:       0x000074DD
+        Context:         0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000B0010000000000033000000000000000000000002020100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040109600000000000100000000000000000000000000000068E7D0C8FF7F000068E7D0C8FF7F000097E6D0C8FF7F000010109600000000000000000000000000020000000000000088E4D0C8FF7F0000603FFF85C77F0000F00340000000000080E7D0C8FF7F000000000000000000000000000000000000E0034000000000007F0300000000000000000000000000000000000000000000801F0000FFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF252525252525252525252525252525250000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000FF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+        Stack:
+          Start of Memory Range: 0x00007FFFC8D0E000
+          Content:               'DEADBEEF'
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s
new file mode 100644
index 000000000000000..d626b4602ad58f4
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s
@@ -0,0 +1,142 @@
+## Test that lldb handles (mainly, that it doesn't crash) the situation where
+## two skeleton compile units have the same DWO ID (and try to claim the same
+## split unit from the DWP file. This can sometimes happen when the compile unit
+## is nearly empty (e.g. because LTO has optimized all of it away).
+
+# RUN: llvm-mc -triple=x86_64-pc-linux -filetype=obj %s --defsym MAIN=0 > %t
+# RUN: llvm-mc -triple=x86_64-pc-linux -filetype=obj %s > %t.dwp
+# RUN: %lldb %t -o "image lookup -t my_enum_type" \
+# RUN:   -o "image dump separate-debug-info" -o exit | FileCheck %s
+
+## Check that we're able to access the type within the split unit (no matter
+## which skeleton unit it ends up associated with). Completely ignoring the unit
+## might also be reasonable.
+# CHECK: image lookup -t my_enum_type
+# CHECK: 1 match found
+# CHECK:      name = "my_enum_type", byte-size = 4, compiler_type = "enum my_enum_type {
+# CHECK-NEXT: }"
+#
+## Check that we get some indication of the error.
+# CHECK: image dump separate-debug-info
+# CHECK:      Dwo ID             Err Dwo Path
+# CHECK: 0xdeadbeefbaadf00d E   multiple compile units with Dwo ID 0xdeadbeefbaadf00d
+
+.set DWO_ID, 0xdeadbeefbaadf00d
+
+## The main file.
+.ifdef MAIN
+        .section        .debug_abbrev,"",@progbits
+        .byte   1                       # Abbreviation Code
+        .byte   74                      # DW_TAG_compile_unit
+        .byte   0                       # DW_CHILDREN_no
+        .byte   0x76                    # DW_AT_dwo_name
+        .byte   8                       # DW_FORM_string
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   0                       # EOM(3)
+
+
+        .section        .debug_info,"",@progbits
+.irpc I,01
+.Lcu_begin\I:
+        .long   .Ldebug_info_end\I-.Ldebug_info_start\I # Length of Unit
+.Ldebug_info_start\I:
+        .short  5                       # DWARF version number
+        .byte   4                       # DWARF Unit Type
+        .byte   8                       # Address Size (in bytes)
+        .long   .debug_abbrev           # Offset Into Abbrev. Section
+        .quad   DWO_ID                  # DWO id
+        .byte   1                       # Abbrev [1] DW_TAG_compile_unit
+        .ascii  "foo"
+        .byte   '0' + \I
+        .asciz  ".dwo\0"                # DW_AT_dwo_name
+.Ldebug_info_end\I:
+.endr
+
+.else
+## DWP file starts here.
+
+        .section        .debug_abbrev.dwo,"e",@progbits
+.LAbbrevBegin:
+        .byte   1                       # Abbreviation Code
+        .byte   17                      # DW_TAG_compile_unit
+        .byte   1                       # DW_CHILDREN_yes
+        .byte   37                      # DW_AT_producer
+        .byte   8                       # DW_FORM_string
+        .byte   19                      # DW_AT_language
+        .byte   5                       # DW_FORM_data2
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   2                       # Abbreviation Code
+        .byte   4                       # DW_TAG_enumeration_type
+        .byte   0                       # DW_CHILDREN_no
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   73                      # DW_AT_type
+        .byte   19                      # DW_FORM_ref4
+        .byte   11                      # DW_AT_byte_size
+        .byte   11                      # DW_FORM_data1
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   4                       # Abbreviation Code
+        .byte   36                      # DW_TAG_base_type
+        .byte   0                       # DW_CHILDREN_no
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   62                      # DW_AT_encoding
+        .byte   11                      # DW_FORM_data1
+        .byte   11                      # DW_AT_byte_size
+        .byte   11                      # DW_FORM_data1
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   0                       # EOM(3)
+.LAbbrevEnd:
+        .section        .debug_info.dwo,"e",@progbits
+.LCUBegin:
+.Lcu_begin1:
+        .long   .Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit
+.Ldebug_info_start1:
+        .short  5                       # DWARF version number
+        .byte   5                       # DWARF Unit Type
+        .byte   8                       # Address Size (in bytes)
+        .long   0                       # Offset Into Abbrev. Section
+        .quad   DWO_ID                  # DWO id
+        .byte   1                       # Abbrev [1] DW_TAG_compile_unit
+        .asciz  "Hand-written DWARF"    # DW_AT_producer
+        .short  12                      # DW_AT_language
+        .byte   2                       # Abbrev [2] DW_TAG_enumeration_type
+        .asciz  "my_enum_type"          # DW_AT_name
+        .long   .Lint-.Lcu_begin1       # DW_AT_type
+        .byte   4                       # DW_AT_byte_size
+.Lint:
+        .byte   4                       # Abbrev [4] DW_TAG_base_type
+        .asciz  "int"                   # DW_AT_name
+        .byte   5                       # DW_AT_encoding
+        .byte   4                       # DW_AT_byte_size
+        .byte   0                       # End Of Children Mark
+.Ldebug_info_end1:
+.LCUEnd:
+        .section .debug_cu_index, "", @progbits
+## Header:
+        .short 5                        # Version
+        .short 0                        # Padding
+        .long 2                         # Section count
+        .long 1                         # Unit count
+        .long 2                         # Slot count
+## Hash Table of Signatures:
+        .quad 0
+        .quad DWO_ID
+## Parallel Table of Indexes:
+        .long 0
+        .long 1
+## Table of Section Offsets:
+## Row 0:
+        .long 1                         # DW_SECT_INFO
+        .long 3                         # DW_SECT_ABBREV
+## Row 1:
+        .long 0                         # Offset in .debug_info.dwo
+        .long 0                         # Offset in .debug_abbrev.dwo
+## Table of Section Sizes:
+        .long .LCUEnd-.LCUBegin         # Size in .debug_info.dwo
+        .long .LAbbrevEnd-.LAbbrevBegin # Size in .debug_abbrev.dwo
+.endif
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index 57562a149835197..7828272aa15a7d6 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -192,8 +192,6 @@ struct DAP {
   std::mutex call_mutex;
   std::map<int /* request_seq */, ResponseCallback /* reply handler */>
       inflight_reverse_requests;
-  StartDebuggingRequestHandler start_debugging_request_handler;
-  ReplModeRequestHandler repl_mode_request_handler;
   ReplMode repl_mode;
   std::string command_escape_prefix = "`";
   lldb::SBFormat frame_format;
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index ea84f31aec3a6c7..f50a6c173107394 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -1627,12 +1627,12 @@ void request_initialize(const llvm::json::Object &request) {
       "lldb-dap", "Commands for managing lldb-dap.");
   if (GetBoolean(arguments, "supportsStartDebuggingRequest", false)) {
     cmd.AddCommand(
-        "startDebugging", &g_dap.start_debugging_request_handler,
+        "startDebugging", new StartDebuggingRequestHandler(),
         "Sends a startDebugging request from the debug adapter to the client "
         "to start a child debug session of the same type as the caller.");
   }
   cmd.AddCommand(
-      "repl-mode", &g_dap.repl_mode_request_handler,
+      "repl-mode", new ReplModeRequestHandler(),
       "Get or set the repl behavior of lldb-dap evaluation requests.");
 
   g_dap.progress_event_thread = std::thread(ProgressEventThreadFunction);
diff --git a/lldb/tools/lldb-dap/package-lock.json b/lldb/tools/lldb-dap/package-lock.json
index 8c70cc2d30e1443..96570e42dbfdc4b 100644
--- a/lldb/tools/lldb-dap/package-lock.json
+++ b/lldb/tools/lldb-dap/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "lldb-dap",
-  "version": "0.2.0",
+  "version": "0.2.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "lldb-dap",
-      "version": "0.2.0",
+      "version": "0.2.4",
       "license": "Apache 2.0 License with LLVM exceptions",
       "devDependencies": {
         "@types/node": "^18.11.18",
diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index 97e4efe7bac19d3..4f4261d1718c018 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -1,7 +1,7 @@
 {
   "name": "lldb-dap",
   "displayName": "LLDB DAP",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "publisher": "llvm-vs-code-extensions",
   "homepage": "https://lldb.llvm.org",
   "description": "LLDB debugging from VSCode",
@@ -73,6 +73,11 @@
           "scope": "resource",
           "type": "string",
           "description": "The path to the lldb-dap binary."
+        },
+        "lldb-dap.log-path": {
+          "scope": "resource",
+          "type": "string",
+          "description": "The log path for lldb-dap (if any)"
         }
       }
     },
diff --git a/lldb/tools/lldb-dap/src-ts/extension.ts b/lldb/tools/lldb-dap/src-ts/extension.ts
index 791175f7b46224e..7df09f7a29dad71 100644
--- a/lldb/tools/lldb-dap/src-ts/extension.ts
+++ b/lldb/tools/lldb-dap/src-ts/extension.ts
@@ -14,13 +14,29 @@ function createDefaultLLDBDapOptions(): LLDBDapOptions {
       session: vscode.DebugSession,
       packageJSONExecutable: vscode.DebugAdapterExecutable | undefined,
     ): Promise<vscode.DebugAdapterExecutable | undefined> {
-      const path = vscode.workspace
-        .getConfiguration("lldb-dap", session.workspaceFolder)
-        .get<string>("executable-path");
+      const config = vscode.workspace
+        .getConfiguration("lldb-dap", session.workspaceFolder);
+      const path = config.get<string>("executable-path");
+      const log_path = config.get<string>("log-path");
+
+      let env : { [key: string]: string } = {};
+      if (log_path) {
+        env["LLDBDAP_LOG"] = log_path;
+      }
+
       if (path) {
-        return new vscode.DebugAdapterExecutable(path, []);
+        return new vscode.DebugAdapterExecutable(path, [], {env});
+      } else if (packageJSONExecutable) {
+        return new vscode.DebugAdapterExecutable(packageJSONExecutable.command, packageJSONExecutable.args, {
+          ...packageJSONExecutable.options,
+          env: {
+            ...packageJSONExecutable.options?.env,
+            ...env
+          }
+        });
+      } else {
+        return undefined;
       }
-      return packageJSONExecutable;
     },
   };
 }
diff --git a/lldb/tools/lldb-server/Acceptor.cpp b/lldb/tools/lldb-server/Acceptor.cpp
index 2037f1e0f62b9a7..cf9a55c6c07da56 100644
--- a/lldb/tools/lldb-server/Acceptor.cpp
+++ b/lldb/tools/lldb-server/Acceptor.cpp
@@ -22,40 +22,6 @@ using namespace lldb_private;
 using namespace lldb_private::lldb_server;
 using namespace llvm;
 
-namespace {
-
-struct SocketScheme {
-  const char *m_scheme;
-  const Socket::SocketProtocol m_protocol;
-};
-
-SocketScheme socket_schemes[] = {
-    {"tcp", Socket::ProtocolTcp},
-    {"udp", Socket::ProtocolUdp},
-    {"unix", Socket::ProtocolUnixDomain},
-    {"unix-abstract", Socket::ProtocolUnixAbstract},
-};
-
-bool FindProtocolByScheme(const char *scheme,
-                          Socket::SocketProtocol &protocol) {
-  for (auto s : socket_schemes) {
-    if (!strcmp(s.m_scheme, scheme)) {
-      protocol = s.m_protocol;
-      return true;
-    }
-  }
-  return false;
-}
-
-const char *FindSchemeByProtocol(const Socket::SocketProtocol protocol) {
-  for (auto s : socket_schemes) {
-    if (s.m_protocol == protocol)
-      return s.m_scheme;
-  }
-  return nullptr;
-}
-}
-
 Status Acceptor::Listen(int backlog) {
   return m_listener_socket_up->Listen(StringRef(m_name), backlog);
 }
@@ -74,7 +40,7 @@ Socket::SocketProtocol Acceptor::GetSocketProtocol() const {
 }
 
 const char *Acceptor::GetSocketScheme() const {
-  return FindSchemeByProtocol(GetSocketProtocol());
+  return Socket::FindSchemeByProtocol(GetSocketProtocol());
 }
 
 std::string Acceptor::GetLocalSocketId() const { return m_local_socket_id(); }
@@ -87,7 +53,8 @@ std::unique_ptr<Acceptor> Acceptor::Create(StringRef name,
   Socket::SocketProtocol socket_protocol = Socket::ProtocolUnixDomain;
   // Try to match socket name as URL - e.g., tcp://localhost:5555
   if (std::optional<URI> res = URI::Parse(name)) {
-    if (!FindProtocolByScheme(res->scheme.str().c_str(), socket_protocol))
+    if (!Socket::FindProtocolByScheme(res->scheme.str().c_str(),
+                                      socket_protocol))
       error.SetErrorStringWithFormat("Unknown protocol scheme \"%s\"",
                                      res->scheme.str().c_str());
     else
diff --git a/lldb/unittests/Callback/TestBreakpointSetCallback.cpp b/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
index 2a7070f9349c029..3dba4a9eb719e3e 100644
--- a/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
+++ b/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
@@ -30,6 +30,8 @@ using namespace lldb;
 static constexpr lldb::user_id_t expected_breakpoint_id = 1;
 static constexpr lldb::user_id_t expected_breakpoint_location_id = 0;
 
+int baton_value;
+
 class BreakpointSetCallbackTest : public ::testing::Test {
 public:
   static void CheckCallbackArgs(void *baton, StoppointCallbackContext *context,
@@ -37,7 +39,7 @@ class BreakpointSetCallbackTest : public ::testing::Test {
                                 lldb::user_id_t break_loc_id,
                                 TargetSP expected_target_sp) {
     EXPECT_EQ(context->exe_ctx_ref.GetTargetSP(), expected_target_sp);
-    EXPECT_EQ(baton, "hello");
+    EXPECT_EQ(baton, &baton_value);
     EXPECT_EQ(break_id, expected_breakpoint_id);
     EXPECT_EQ(break_loc_id, expected_breakpoint_location_id);
   }
@@ -55,7 +57,6 @@ class BreakpointSetCallbackTest : public ::testing::Test {
 };
 
 TEST_F(BreakpointSetCallbackTest, TestBreakpointSetCallback) {
-  void *baton = (void *)"hello";
   // Set up the debugger, make sure that was done properly.
   TargetSP target_sp;
   ArchSpec arch("x86_64-apple-macosx-");
@@ -78,7 +79,7 @@ TEST_F(BreakpointSetCallbackTest, TestBreakpointSetCallback) {
         CheckCallbackArgs(baton, context, break_id, break_loc_id, target_sp);
         return true;
       },
-      baton, true);
+      (void *)&baton_value, true);
   ExecutionContext exe_ctx(target_sp, false);
   StoppointCallbackContext context(nullptr, exe_ctx, true);
   breakpoint_sp->InvokeCallback(&context, 0);
diff --git a/lldb/unittests/Core/CMakeLists.txt b/lldb/unittests/Core/CMakeLists.txt
index d40c357e3f463be..949963fd4034639 100644
--- a/lldb/unittests/Core/CMakeLists.txt
+++ b/lldb/unittests/Core/CMakeLists.txt
@@ -3,7 +3,6 @@ add_lldb_unittest(LLDBCoreTests
   DiagnosticEventTest.cpp
   DumpDataExtractorTest.cpp
   DumpRegisterInfoTest.cpp
-  FileSpecListTest.cpp
   FormatEntityTest.cpp
   MangledTest.cpp
   ModuleSpecTest.cpp
diff --git a/lldb/unittests/Core/FileSpecListTest.cpp b/lldb/unittests/Core/FileSpecListTest.cpp
deleted file mode 100644
index e63f4a00bc3a944..000000000000000
--- a/lldb/unittests/Core/FileSpecListTest.cpp
+++ /dev/null
@@ -1,125 +0,0 @@
-//===-- FileSpecListTest.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "gtest/gtest.h"
-
-#include "lldb/Utility/FileSpecList.h"
-
-using namespace lldb_private;
-
-static FileSpec PosixSpec(llvm::StringRef path) {
-  return FileSpec(path, FileSpec::Style::posix);
-}
-
-static FileSpec WindowsSpec(llvm::StringRef path) {
-  return FileSpec(path, FileSpec::Style::windows);
-}
-
-TEST(SupportFileListTest, RelativePathMatchesPosix) {
-
-  const FileSpec fullpath = PosixSpec("/build/src/main.cpp");
-  const FileSpec relative = PosixSpec("./src/main.cpp");
-  const FileSpec basename = PosixSpec("./main.cpp");
-  const FileSpec full_wrong = PosixSpec("/other/wrong/main.cpp");
-  const FileSpec rel_wrong = PosixSpec("./wrong/main.cpp");
-  // Make sure these don't match "src/main.cpp" as we want to match full
-  // directories only
-  const FileSpec rel2_wrong = PosixSpec("asrc/main.cpp");
-  const FileSpec rel3_wrong = PosixSpec("rc/main.cpp");
-
-  SupportFileList files;
-  files.Append(fullpath);
-  files.Append(relative);
-  files.Append(basename);
-  files.Append(full_wrong);
-  files.Append(rel_wrong);
-  files.Append(rel2_wrong);
-  files.Append(rel3_wrong);
-
-  // Make sure the full path only matches the first entry
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, fullpath));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, fullpath));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, fullpath));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, fullpath));
-  // Make sure the relative path matches the all of the entries that contain
-  // the relative path
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, relative));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, relative));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, relative));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, relative));
-
-  // Make sure looking file a file using the basename matches all entries
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, basename));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, basename));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, basename));
-  EXPECT_EQ((size_t)3, files.FindCompatibleIndex(3, basename));
-  EXPECT_EQ((size_t)4, files.FindCompatibleIndex(4, basename));
-  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(5, basename));
-  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(6, basename));
-
-  // Make sure that paths that have a common suffix don't return values that
-  // don't match on directory delimiters.
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel2_wrong));
-  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(3, rel2_wrong));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(6, rel2_wrong));
-
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel3_wrong));
-  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong));
-}
-
-TEST(SupportFileListTest, RelativePathMatchesWindows) {
-
-  const FileSpec fullpath = WindowsSpec(R"(C:\build\src\main.cpp)");
-  const FileSpec relative = WindowsSpec(R"(.\src\main.cpp)");
-  const FileSpec basename = WindowsSpec(R"(.\main.cpp)");
-  const FileSpec full_wrong = WindowsSpec(R"(\other\wrong\main.cpp)");
-  const FileSpec rel_wrong = WindowsSpec(R"(.\wrong\main.cpp)");
-  // Make sure these don't match "src\main.cpp" as we want to match full
-  // directories only
-  const FileSpec rel2_wrong = WindowsSpec(R"(asrc\main.cpp)");
-  const FileSpec rel3_wrong = WindowsSpec(R"("rc\main.cpp)");
-
-  SupportFileList files;
-  files.Append(fullpath);
-  files.Append(relative);
-  files.Append(basename);
-  files.Append(full_wrong);
-  files.Append(rel_wrong);
-  files.Append(rel2_wrong);
-  files.Append(rel3_wrong);
-
-  // Make sure the full path only matches the first entry
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, fullpath));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, fullpath));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, fullpath));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, fullpath));
-  // Make sure the relative path matches the all of the entries that contain
-  // the relative path
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, relative));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, relative));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, relative));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, relative));
-
-  // Make sure looking file a file using the basename matches all entries
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, basename));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, basename));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, basename));
-  EXPECT_EQ((size_t)3, files.FindCompatibleIndex(3, basename));
-  EXPECT_EQ((size_t)4, files.FindCompatibleIndex(4, basename));
-  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(5, basename));
-  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(6, basename));
-
-  // Make sure that paths that have a common suffix don't return values that
-  // don't match on directory delimiters.
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel2_wrong));
-  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(3, rel2_wrong));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(6, rel2_wrong));
-
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel3_wrong));
-  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong));
-}
diff --git a/lldb/unittests/Disassembler/x86/TestGetControlFlowKindx86.cpp b/lldb/unittests/Disassembler/x86/TestGetControlFlowKindx86.cpp
index fecd93361e56338..e9d5ae3f7b3cdba 100644
--- a/lldb/unittests/Disassembler/x86/TestGetControlFlowKindx86.cpp
+++ b/lldb/unittests/Disassembler/x86/TestGetControlFlowKindx86.cpp
@@ -145,14 +145,19 @@ TEST_F(TestGetControlFlowKindx86, TestX86_64Instruction) {
     EXPECT_EQ(kind, result[i]);
 
     // Also, test the DisassemblerLLVMC::MCDisasmInstance methods.
-    if (kind == eInstructionControlFlowKindReturn)
+    if (kind == eInstructionControlFlowKindReturn) {
       EXPECT_FALSE(inst_sp->IsCall());
-    if (kind == eInstructionControlFlowKindCall)
+    }
+
+    if (kind == eInstructionControlFlowKindCall) {
       EXPECT_TRUE(inst_sp->IsCall());
+    }
+
     if (kind == eInstructionControlFlowKindCall ||
         kind == eInstructionControlFlowKindJump ||
         kind == eInstructionControlFlowKindCondJump ||
-        kind == eInstructionControlFlowKindReturn)
+        kind == eInstructionControlFlowKindReturn) {
       EXPECT_TRUE(inst_sp->DoesBranch());
+    }
   }
 }
diff --git a/lldb/unittests/Process/CMakeLists.txt b/lldb/unittests/Process/CMakeLists.txt
index 5fbecfcfa25ebe1..d3b37e006fd899a 100644
--- a/lldb/unittests/Process/CMakeLists.txt
+++ b/lldb/unittests/Process/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_subdirectory(gdb-remote)
 if (CMAKE_SYSTEM_NAME MATCHES "Linux|Android")
+  add_subdirectory(elf-core)
   add_subdirectory(Linux)
   add_subdirectory(POSIX)
 endif()
diff --git a/lldb/unittests/Process/elf-core/CMakeLists.txt b/lldb/unittests/Process/elf-core/CMakeLists.txt
new file mode 100644
index 000000000000000..b852a3ffb863c1d
--- /dev/null
+++ b/lldb/unittests/Process/elf-core/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_lldb_unittest(ProcessElfCoreTests
+  ThreadElfCoreTest.cpp
+
+  LINK_LIBS
+    lldbCore
+    lldbHost
+    lldbUtilityHelpers
+    lldbPluginProcessElfCore
+    lldbPluginPlatformLinux
+
+    LLVMTestingSupport
+
+  LINK_COMPONENTS
+    Support
+  )
diff --git a/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp b/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp
new file mode 100644
index 000000000000000..3bc8b9053d20091
--- /dev/null
+++ b/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp
@@ -0,0 +1,178 @@
+//===-- ThreadElfCoreTest.cpp ------------------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "Plugins/Process/elf-core/ThreadElfCore.h"
+#include "Plugins/Platform/Linux/PlatformLinux.h"
+#include "TestingSupport/TestUtilities.h"
+#include "lldb/Core/Debugger.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Host/HostInfo.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Target/Target.h"
+#include "lldb/Target/Thread.h"
+#include "lldb/Utility/Listener.h"
+#include "llvm/TargetParser/Triple.h"
+#include "gtest/gtest.h"
+
+#include <memory>
+#include <mutex>
+#include <sys/resource.h>
+#include <unistd.h>
+
+using namespace lldb_private;
+
+namespace {
+
+struct ElfCoreTest : public testing::Test {
+  static void SetUpTestCase() {
+    FileSystem::Initialize();
+    HostInfo::Initialize();
+    platform_linux::PlatformLinux::Initialize();
+    std::call_once(TestUtilities::g_debugger_initialize_flag,
+                   []() { Debugger::Initialize(nullptr); });
+  }
+  static void TearDownTestCase() {
+    platform_linux::PlatformLinux::Terminate();
+    HostInfo::Terminate();
+    FileSystem::Terminate();
+  }
+};
+
+struct DummyProcess : public Process {
+  DummyProcess(lldb::TargetSP target_sp, lldb::ListenerSP listener_sp)
+      : Process(target_sp, listener_sp) {
+    SetID(getpid());
+  }
+
+  bool CanDebug(lldb::TargetSP target, bool plugin_specified_by_name) override {
+    return true;
+  }
+  Status DoDestroy() override { return {}; }
+  void RefreshStateAfterStop() override {}
+  size_t DoReadMemory(lldb::addr_t vm_addr, void *buf, size_t size,
+                      Status &error) override {
+    return 0;
+  }
+  bool DoUpdateThreadList(ThreadList &old_thread_list,
+                          ThreadList &new_thread_list) override {
+    return false;
+  }
+  llvm::StringRef GetPluginName() override { return "Dummy"; }
+};
+
+struct DummyThread : public Thread {
+  using Thread::Thread;
+
+  ~DummyThread() override { DestroyThread(); }
+
+  void RefreshStateAfterStop() override {}
+
+  lldb::RegisterContextSP GetRegisterContext() override { return nullptr; }
+
+  lldb::RegisterContextSP
+  CreateRegisterContextForFrame(StackFrame *frame) override {
+    return nullptr;
+  }
+
+  bool CalculateStopInfo() override { return false; }
+};
+
+lldb::TargetSP CreateTarget(lldb::DebuggerSP &debugger_sp, ArchSpec &arch) {
+  lldb::PlatformSP platform_sp;
+  lldb::TargetSP target_sp;
+  debugger_sp->GetTargetList().CreateTarget(
+      *debugger_sp, "", arch, eLoadDependentsNo, platform_sp, target_sp);
+  return target_sp;
+}
+
+lldb::ThreadSP CreateThread(lldb::ProcessSP &process_sp) {
+  lldb::ThreadSP thread_sp =
+      std::make_shared<DummyThread>(*process_sp.get(), gettid());
+  if (thread_sp == nullptr) {
+    return nullptr;
+  }
+  process_sp->GetThreadList().AddThread(thread_sp);
+
+  return thread_sp;
+}
+
+} // namespace
+
+TEST_F(ElfCoreTest, PopulatePrpsInfoTest) {
+  ArchSpec arch{HostInfo::GetTargetTriple()};
+  lldb::DebuggerSP debugger_sp = Debugger::CreateInstance();
+  ASSERT_TRUE(debugger_sp);
+
+  lldb::TargetSP target_sp = CreateTarget(debugger_sp, arch);
+  ASSERT_TRUE(target_sp);
+
+  lldb::ListenerSP listener_sp(Listener::MakeListener("dummy"));
+  lldb::ProcessSP process_sp =
+      std::make_shared<DummyProcess>(target_sp, listener_sp);
+  ASSERT_TRUE(process_sp);
+  auto prpsinfo_opt = ELFLinuxPrPsInfo::Populate(process_sp);
+  ASSERT_TRUE(prpsinfo_opt.has_value());
+  ASSERT_EQ(prpsinfo_opt->pr_pid, getpid());
+  ASSERT_EQ(prpsinfo_opt->pr_state, 0);
+  ASSERT_EQ(prpsinfo_opt->pr_sname, 'R');
+  ASSERT_EQ(prpsinfo_opt->pr_zomb, 0);
+  int priority = getpriority(PRIO_PROCESS, getpid());
+  if (priority == -1)
+    ASSERT_EQ(errno, 0);
+  ASSERT_EQ(prpsinfo_opt->pr_nice, priority);
+  ASSERT_EQ(prpsinfo_opt->pr_flag, 0UL);
+  ASSERT_EQ(prpsinfo_opt->pr_uid, getuid());
+  ASSERT_EQ(prpsinfo_opt->pr_gid, getgid());
+  ASSERT_EQ(prpsinfo_opt->pr_pid, getpid());
+  ASSERT_EQ(prpsinfo_opt->pr_ppid, getppid());
+  ASSERT_EQ(prpsinfo_opt->pr_pgrp, getpgrp());
+  ASSERT_EQ(prpsinfo_opt->pr_sid, getsid(getpid()));
+  ASSERT_EQ(std::string{prpsinfo_opt->pr_fname}, "ProcessElfCoreT");
+  ASSERT_TRUE(std::string{prpsinfo_opt->pr_psargs}.empty());
+  lldb_private::ProcessInstanceInfo info;
+  ASSERT_TRUE(process_sp->GetProcessInfo(info));
+  const char *args[] = {"a.out", "--foo=bar", "--baz=boo", nullptr};
+  info.SetArguments(args, true);
+  prpsinfo_opt =
+      ELFLinuxPrPsInfo::Populate(info, lldb::StateType::eStateStopped);
+  ASSERT_TRUE(prpsinfo_opt.has_value());
+  ASSERT_EQ(prpsinfo_opt->pr_pid, getpid());
+  ASSERT_EQ(prpsinfo_opt->pr_state, 3);
+  ASSERT_EQ(prpsinfo_opt->pr_sname, 'T');
+  ASSERT_EQ(std::string{prpsinfo_opt->pr_fname}, "a.out");
+  ASSERT_FALSE(std::string{prpsinfo_opt->pr_psargs}.empty());
+  ASSERT_EQ(std::string{prpsinfo_opt->pr_psargs}, "a.out --foo=bar --baz=boo");
+}
+
+TEST_F(ElfCoreTest, PopulatePrStatusTest) {
+  ArchSpec arch{HostInfo::GetTargetTriple()};
+  lldb::DebuggerSP debugger_sp = Debugger::CreateInstance();
+  ASSERT_TRUE(debugger_sp);
+
+  lldb::TargetSP target_sp = CreateTarget(debugger_sp, arch);
+  ASSERT_TRUE(target_sp);
+
+  lldb::ListenerSP listener_sp(Listener::MakeListener("dummy"));
+  lldb::ProcessSP process_sp =
+      std::make_shared<DummyProcess>(target_sp, listener_sp);
+  ASSERT_TRUE(process_sp);
+  lldb::ThreadSP thread_sp = CreateThread(process_sp);
+  ASSERT_TRUE(thread_sp);
+  auto prstatus_opt = ELFLinuxPrStatus::Populate(thread_sp);
+  ASSERT_TRUE(prstatus_opt.has_value());
+  ASSERT_EQ(prstatus_opt->si_signo, 0);
+  ASSERT_EQ(prstatus_opt->si_code, 0);
+  ASSERT_EQ(prstatus_opt->si_errno, 0);
+  ASSERT_EQ(prstatus_opt->pr_cursig, 0);
+  ASSERT_EQ(prstatus_opt->pr_sigpend, 0UL);
+  ASSERT_EQ(prstatus_opt->pr_sighold, 0UL);
+  ASSERT_EQ(prstatus_opt->pr_pid, static_cast<uint32_t>(gettid()));
+  ASSERT_EQ(prstatus_opt->pr_ppid, static_cast<uint32_t>(getppid()));
+  ASSERT_EQ(prstatus_opt->pr_pgrp, static_cast<uint32_t>(getpgrp()));
+  ASSERT_EQ(prstatus_opt->pr_sid, static_cast<uint32_t>(getsid(gettid())));
+}
diff --git a/lldb/unittests/Utility/CMakeLists.txt b/lldb/unittests/Utility/CMakeLists.txt
index 8e12815d51541c9..40e0959fc01d142 100644
--- a/lldb/unittests/Utility/CMakeLists.txt
+++ b/lldb/unittests/Utility/CMakeLists.txt
@@ -12,6 +12,7 @@ add_lldb_unittest(UtilityTests
   DataExtractorTest.cpp
   EnvironmentTest.cpp
   EventTest.cpp
+  FileSpecListTest.cpp
   FileSpecTest.cpp
   FlagsTest.cpp
   ListenerTest.cpp
@@ -22,6 +23,7 @@ add_lldb_unittest(UtilityTests
   ProcessInstanceInfoTest.cpp
   RangeMapTest.cpp
   RangeTest.cpp
+  RealpathPrefixesTest.cpp
   RegisterValueTest.cpp
   RegularExpressionTest.cpp
   ScalarTest.cpp
@@ -49,6 +51,7 @@ add_lldb_unittest(UtilityTests
   XcodeSDKTest.cpp
 
   LINK_LIBS
+      lldbTarget
       lldbUtility
       lldbUtilityHelpers
       LLVMTestingSupport
diff --git a/lldb/unittests/Utility/FileSpecListTest.cpp b/lldb/unittests/Utility/FileSpecListTest.cpp
new file mode 100644
index 000000000000000..d3f89ad0dfcb3c6
--- /dev/null
+++ b/lldb/unittests/Utility/FileSpecListTest.cpp
@@ -0,0 +1,330 @@
+//===-- FileSpecListTest.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include "MockSymlinkFileSystem.h"
+#include "lldb/Utility/FileSpecList.h"
+#include "lldb/Utility/RealpathPrefixes.h"
+
+using namespace lldb_private;
+
+static FileSpec PosixSpec(llvm::StringRef path) {
+  return FileSpec(path, FileSpec::Style::posix);
+}
+
+static FileSpec WindowsSpec(llvm::StringRef path) {
+  return FileSpec(path, FileSpec::Style::windows);
+}
+
+TEST(SupportFileListTest, RelativePathMatchesPosix) {
+
+  const FileSpec fullpath = PosixSpec("/build/src/main.cpp");
+  const FileSpec relative = PosixSpec("./src/main.cpp");
+  const FileSpec basename = PosixSpec("./main.cpp");
+  const FileSpec full_wrong = PosixSpec("/other/wrong/main.cpp");
+  const FileSpec rel_wrong = PosixSpec("./wrong/main.cpp");
+  // Make sure these don't match "src/main.cpp" as we want to match full
+  // directories only
+  const FileSpec rel2_wrong = PosixSpec("asrc/main.cpp");
+  const FileSpec rel3_wrong = PosixSpec("rc/main.cpp");
+
+  SupportFileList files;
+  files.Append(fullpath);
+  files.Append(relative);
+  files.Append(basename);
+  files.Append(full_wrong);
+  files.Append(rel_wrong);
+  files.Append(rel2_wrong);
+  files.Append(rel3_wrong);
+
+  // Make sure the full path only matches the first entry
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, fullpath));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, fullpath));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, fullpath));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, fullpath));
+  // Make sure the relative path matches the all of the entries that contain
+  // the relative path
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, relative));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, relative));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, relative));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, relative));
+
+  // Make sure looking file a file using the basename matches all entries
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, basename));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, basename));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, basename));
+  EXPECT_EQ((size_t)3, files.FindCompatibleIndex(3, basename));
+  EXPECT_EQ((size_t)4, files.FindCompatibleIndex(4, basename));
+  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(5, basename));
+  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(6, basename));
+
+  // Make sure that paths that have a common suffix don't return values that
+  // don't match on directory delimiters.
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel2_wrong));
+  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(3, rel2_wrong));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(6, rel2_wrong));
+
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel3_wrong));
+  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong));
+}
+
+TEST(SupportFileListTest, RelativePathMatchesWindows) {
+
+  const FileSpec fullpath = WindowsSpec(R"(C:\build\src\main.cpp)");
+  const FileSpec relative = WindowsSpec(R"(.\src\main.cpp)");
+  const FileSpec basename = WindowsSpec(R"(.\main.cpp)");
+  const FileSpec full_wrong = WindowsSpec(R"(\other\wrong\main.cpp)");
+  const FileSpec rel_wrong = WindowsSpec(R"(.\wrong\main.cpp)");
+  // Make sure these don't match "src\main.cpp" as we want to match full
+  // directories only
+  const FileSpec rel2_wrong = WindowsSpec(R"(asrc\main.cpp)");
+  const FileSpec rel3_wrong = WindowsSpec(R"("rc\main.cpp)");
+
+  SupportFileList files;
+  files.Append(fullpath);
+  files.Append(relative);
+  files.Append(basename);
+  files.Append(full_wrong);
+  files.Append(rel_wrong);
+  files.Append(rel2_wrong);
+  files.Append(rel3_wrong);
+
+  // Make sure the full path only matches the first entry
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, fullpath));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, fullpath));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, fullpath));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, fullpath));
+  // Make sure the relative path matches the all of the entries that contain
+  // the relative path
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, relative));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, relative));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, relative));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, relative));
+
+  // Make sure looking file a file using the basename matches all entries
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, basename));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, basename));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, basename));
+  EXPECT_EQ((size_t)3, files.FindCompatibleIndex(3, basename));
+  EXPECT_EQ((size_t)4, files.FindCompatibleIndex(4, basename));
+  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(5, basename));
+  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(6, basename));
+
+  // Make sure that paths that have a common suffix don't return values that
+  // don't match on directory delimiters.
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel2_wrong));
+  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(3, rel2_wrong));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(6, rel2_wrong));
+
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel3_wrong));
+  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong));
+}
+
+// Support file is a symlink to the breakpoint file.
+// Absolute paths are used.
+// A matching prefix is set.
+// Should find it compatible.
+TEST(SupportFileListTest, SymlinkedAbsolutePaths) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("/symlink_dir/foo.h"), PosixSpec("/real_dir/foo.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("/symlink_dir"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("/symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("/real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, (size_t)0);
+}
+
+// Support file is a symlink to the breakpoint file.
+// Absolute paths are used.
+// A matching prefix is set, which is the root directory.
+// Should find it compatible.
+TEST(SupportFileListTest, RootDirectory) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("/symlink_dir/foo.h"), PosixSpec("/real_dir/foo.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("/"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("/symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("/real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, (size_t)0);
+}
+
+// Support file is a symlink to the breakpoint file.
+// Relative paths are used.
+// A matching prefix is set.
+// Should find it compatible.
+TEST(SupportFileListTest, SymlinkedRelativePaths) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("symlink_dir/foo.h"), PosixSpec("real_dir/foo.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("symlink_dir"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, (size_t)0);
+}
+
+// Support file is a symlink to the breakpoint file.
+// A matching prefix is set.
+// Input file only match basename and not directory.
+// Should find it incompatible.
+TEST(SupportFileListTest, RealpathOnlyMatchFileName) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("symlink_dir/foo.h"), PosixSpec("real_dir/foo.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("symlink_dir"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("some_other_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, UINT32_MAX);
+}
+
+// Support file is a symlink to the breakpoint file.
+// A prefix is set, which is a matching string prefix, but not a path prefix.
+// Should find it incompatible.
+TEST(SupportFileListTest, DirectoryMatchStringPrefixButNotWholeDirectoryName) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("symlink_dir/foo.h"), PosixSpec("real_dir/foo.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("symlink")); // This is a string prefix of the
+                                               // symlink but not a path prefix.
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, UINT32_MAX);
+}
+
+// Support file is a symlink to the breakpoint file.
+// A matching prefix is set.
+// However, the breakpoint is set with a partial path.
+// Should find it compatible.
+TEST(SupportFileListTest, PartialBreakpointPath) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("symlink_dir/foo.h"), PosixSpec("/real_dir/foo.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("symlink_dir"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, (size_t)0);
+}
+
+// Support file is a symlink to the breakpoint file.
+// A matching prefix is set.
+// However, the basename is different between the symlink and its target.
+// Should find it incompatible.
+TEST(SupportFileListTest, DifferentBasename) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("/symlink_dir/foo.h"), PosixSpec("/real_dir/bar.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("/symlink_dir"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("/symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("real_dir/bar.h"), &prefixes);
+  EXPECT_EQ(ret, UINT32_MAX);
+}
+
+// No prefixes are configured.
+// The support file and the breakpoint file are different.
+// Should find it incompatible.
+TEST(SupportFileListTest, NoPrefixes) {
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("/real_dir/bar.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("/real_dir/foo.h"), nullptr);
+  EXPECT_EQ(ret, UINT32_MAX);
+}
+
+// No prefixes are configured.
+// The support file and the breakpoint file are the same.
+// Should find it compatible.
+TEST(SupportFileListTest, SameFile) {
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.Append(PosixSpec("/real_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, PosixSpec("/real_dir/foo.h"), nullptr);
+  EXPECT_EQ(ret, (size_t)0);
+}
diff --git a/lldb/unittests/Utility/MockSymlinkFileSystem.h b/lldb/unittests/Utility/MockSymlinkFileSystem.h
new file mode 100644
index 000000000000000..7fa1f93bfa38a9a
--- /dev/null
+++ b/lldb/unittests/Utility/MockSymlinkFileSystem.h
@@ -0,0 +1,66 @@
+//===-- MockSymlinkFileSystem.h
+//--------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Utility/FileSpec.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+namespace lldb_private {
+
+// A mock file system that realpath's a given symlink to a given realpath.
+class MockSymlinkFileSystem : public llvm::vfs::FileSystem {
+public:
+  // Treat all files as non-symlinks.
+  MockSymlinkFileSystem() = default;
+
+  /// Treat \a symlink as a symlink to \a realpath. Treat all other files as
+  /// non-symlinks.
+  MockSymlinkFileSystem(FileSpec &&symlink, FileSpec &&realpath,
+                        FileSpec::Style style = FileSpec::Style::native)
+      : m_symlink(std::move(symlink)), m_realpath(std::move(realpath)),
+        m_style(style) {}
+
+  /// If \a Path matches the symlink given in the ctor, put the realpath given
+  /// in the ctor into \a Output.
+  std::error_code getRealPath(const llvm::Twine &Path,
+                              llvm::SmallVectorImpl<char> &Output) override {
+    if (FileSpec(Path.str(), m_style) == m_symlink) {
+      std::string path = m_realpath.GetPath();
+      Output.assign(path.begin(), path.end());
+    } else {
+      Path.toVector(Output);
+    }
+    return {};
+  }
+
+  // Implement the rest of the interface
+  llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
+    return llvm::errc::operation_not_permitted;
+  }
+  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
+  openFileForRead(const llvm::Twine &Path) override {
+    return llvm::errc::operation_not_permitted;
+  }
+  llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
+                                          std::error_code &EC) override {
+    return llvm::vfs::directory_iterator();
+  }
+  std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
+    return llvm::errc::operation_not_permitted;
+  }
+  llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
+    return llvm::errc::operation_not_permitted;
+  }
+
+private:
+  FileSpec m_symlink;
+  FileSpec m_realpath;
+  FileSpec::Style m_style;
+};
+
+} // namespace lldb_private
diff --git a/lldb/unittests/Utility/RealpathPrefixesTest.cpp b/lldb/unittests/Utility/RealpathPrefixesTest.cpp
new file mode 100644
index 000000000000000..a4dc7d6dd9822df
--- /dev/null
+++ b/lldb/unittests/Utility/RealpathPrefixesTest.cpp
@@ -0,0 +1,153 @@
+//===-- RealpathPrefixesTest.cpp
+//--------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include "MockSymlinkFileSystem.h"
+#include "lldb/Utility/FileSpecList.h"
+#include "lldb/Utility/RealpathPrefixes.h"
+
+using namespace lldb_private;
+
+static FileSpec PosixSpec(llvm::StringRef path) {
+  return FileSpec(path, FileSpec::Style::posix);
+}
+
+static FileSpec WindowsSpec(llvm::StringRef path) {
+  return FileSpec(path, FileSpec::Style::windows);
+}
+
+// Should resolve a symlink which match an absolute prefix
+TEST(RealpathPrefixesTest, MatchingAbsolutePrefix) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("/dir1/link.h"), PosixSpec("/dir2/real.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("/dir1"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(PosixSpec("/dir1/link.h"));
+  EXPECT_EQ(ret, PosixSpec("/dir2/real.h"));
+}
+
+// Should resolve a symlink which match a relative prefix
+TEST(RealpathPrefixesTest, MatchingRelativePrefix) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("dir1/link.h"), PosixSpec("dir2/real.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("dir1"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(PosixSpec("dir1/link.h"));
+  EXPECT_EQ(ret, PosixSpec("dir2/real.h"));
+}
+
+// Should resolve in Windows and/or with a case-insensitive support file
+TEST(RealpathPrefixesTest, WindowsAndCaseInsensitive) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      WindowsSpec("f:\\dir1\\link.h"), WindowsSpec("f:\\dir2\\real.h"),
+      FileSpec::Style::windows));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(WindowsSpec("f:\\dir1"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(WindowsSpec("F:\\DIR1\\LINK.H"));
+  EXPECT_EQ(ret, WindowsSpec("f:\\dir2\\real.h"));
+}
+
+// Should resolve a symlink when there is mixture of matching and mismatching
+// prefixex
+TEST(RealpathPrefixesTest, MatchingAndMismatchingPrefix) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("/dir1/link.h"), PosixSpec("/dir2/real.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("/fake/path1"));
+  file_spec_list.Append(PosixSpec("/dir1")); // Matching prefix
+  file_spec_list.Append(PosixSpec("/fake/path2"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(PosixSpec("/dir1/link.h"));
+  EXPECT_EQ(ret, PosixSpec("/dir2/real.h"));
+}
+
+// Should resolve a symlink when the prefixes matches after normalization
+TEST(RealpathPrefixesTest, ComplexPrefixes) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("dir1/link.h"), PosixSpec("dir2/real.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(
+      PosixSpec("./dir1/foo/../bar/..")); // Equivalent to "/dir1"
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(PosixSpec("dir1/link.h"));
+  EXPECT_EQ(ret, PosixSpec("dir2/real.h"));
+}
+
+// Should not resolve a symlink which doesn't match any prefixes
+TEST(RealpathPrefixesTest, MismatchingPrefixes) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      PosixSpec("/dir1/link.h"), PosixSpec("/dir2/real.h"),
+      FileSpec::Style::posix));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("/dir3"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(PosixSpec("/dir1/link.h"));
+  EXPECT_EQ(ret, std::nullopt);
+}
+
+// Should not resolve a realpath
+TEST(RealpathPrefixesTest, Realpath) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(
+      new MockSymlinkFileSystem());
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.Append(PosixSpec("/symlink_dir"));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(PosixSpec("/dir/real.h"));
+  EXPECT_EQ(ret, std::nullopt);
+}
diff --git a/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst b/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
index 7d4e06ebf0493c4..b71e80f922250b9 100644
--- a/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
+++ b/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
@@ -3804,6 +3804,12 @@ A.5.1 Base Type Entries
       entry, or allow some forms of ``DW_TAG_array_type`` (those that have the
       ``DW_AT_GNU_vector`` attribute) to be used as stack entry value types?
 
+2. A ``DW_TAG_base_type`` debugger information entry with the encoding
+   ``DW_ATE_address`` may have a ``DW_AT_LLVM_address_space`` attribute whose
+   value is an architecture specific address space (see
+   :ref:`amdgpu-dwarf-address-spaces`). If ommitted it defaults to
+   ``DW_ASPACE_LLVM_none``.
+
 .. _amdgpu-dwarf-type-modifier-entries:
 
 A.5.3 Type Modifier Entries
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 117fc2cf6bbbc89..80140734cbefd64 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2378,6 +2378,9 @@ are deprecated and should not be used.
      ======== ============================== ======================================
      "AMDGPU" ``NT_AMDGPU_METADATA``         Metadata in Message Pack [MsgPack]_
                                              binary format.
+     "AMDGPU" ``NT_AMDGPU_KFD_CORE_STATE``   Snapshot of runtime, agent and queues
+                                             state for use in core dump.  See
+                                             :ref:`amdgpu_corefile_note`.
      ======== ============================== ======================================
 
 ..
@@ -2390,6 +2393,7 @@ are deprecated and should not be used.
      ============================== =====
      *reserved*                     0-31
      ``NT_AMDGPU_METADATA``         32
+     ``NT_AMDGPU_KFD_CORE_STATE``   33
      ============================== =====
 
 ``NT_AMDGPU_METADATA``
@@ -2914,6 +2918,34 @@ controlled by the execution mask. An undefined location description together
 with ``DW_OP_LLVM_extend`` is used to indicate the lane was not active on entry
 to the subprogram. See :ref:`amdgpu-dwarf-dw-at-llvm-lane-pc` for an example.
 
+.. _amdgpu-dwarf-base-type-conversions:
+
+Base Type Conversions
+---------------------
+
+For AMDGPU expressions, ``DW_OP_convert`` may be used to convert between
+``DW_ATE_address``-encoded base types in different address spaces.
+
+Conversions are defined as in :ref:`amdgpu-address-spaces` when all relevant
+conditions described there are met, and otherwise result in an evaluation
+error.
+
+.. note::
+
+  For a target which does not support a particular address space, converting to
+  or from that address space is always an evaluation error.
+
+  For targets which support the generic address space, converting from
+  ``DW_ASPACE_AMDGPU_generic`` to ``DW_ASPACE_LLVM_none`` is always defined and
+  requires no change to the literal value of the address.
+
+  Converting from ``DW_ASPACE_AMDGPU_generic`` to any of
+  ``DW_ASPACE_AMDGPU_local``, ``DW_ASPACE_AMDGPU_private_wave`` or
+  ``DW_ASPACE_AMDGPU_private_lane`` is defined when the relevant hardware
+  support is present and setup has been completed. Conversion to
+  ``DW_ASPACE_AMDGPU_private_lane`` additionally requires the context to
+  include the active lane.
+
 Debugger Information Entry Attributes
 -------------------------------------
 
@@ -15024,6 +15056,115 @@ instructions are handled as follows:
                                      trap handler installed.
      =============== =============== ===========================================
 
+Core file format
+================
+
+This section describes the format of core files supporting AMDGPU. Core dumps
+for an AMDGPU program can come in 2 flavors: split or unified core files.
+
+The split layout consists of one host core file containing the information to
+rebuild the image of the host process and one AMDGPU core file that contains
+the information for the AMDGPU agents used in the process.  The AMDGPU core
+file consists of:
+
+* A note describing the state of the AMDGPU agents, AMDGPU queues, and AMDGPU
+  runtime for the process (see :ref:`amdgpu_corefile_note`).
+* A list of load segments containing an image of the AMDGPU agents' memory (see
+  :ref:`amdgpu_corefile_memory`).
+
+The unified core file is the union of all the information contained in
+the two files of the split layout (all notes and load segments).  It contains
+all the information required to reconstruct the image of the process across all
+the agents.
+
+Core file header
+----------------
+
+An AMDGPU core file is an ``ELF64`` core file.  The content of the header
+differs in unified core file layout and AMDGPU core file layout.
+
+Split files
+~~~~~~~~~~~
+
+In the split files layout, the AMDGPU core file is an ``ELF64`` file with the
+header configured as described in :ref:`amdgpu-corefile-headers-table`:
+
+  .. table:: AMDGPU corefile headers
+     :name: amdgpu-corefile-headers-table
+
+     ========================== ===================================
+     Field                      Value
+     ========================== ===================================
+     ``e_ident[EI_CLASS]``      ``ELFCLASS64`` (``0x2``)
+     ``e_ident[EI_DATA]``       ``ELFDATA2LSB`` (``0x1``)
+     ``e_ident[EI_OSABI]``      ``ELFOSABI_AMDGPU_HSA`` (``0x40``)
+     ``e_type``                 ``ET_CORE``(``0x4``)
+     ``e_ident[EI_ABIVERSION]`` ``ELFABIVERSION_AMDGPU_HSA_5``
+     ``e_machine``              ``EM_AMDGPU`` (``0xe0``)
+     ========================== ===================================
+
+Unified file
+~~~~~~~~~~~~
+
+In the unified core file mode, the ``ELF64`` headers are set to describe
+the host architecture and process.
+
+.. _amdgpu_corefile_note:
+
+Core file notes
+---------------
+
+An AMDGPU core file must contain one snapshot note in a ``PT_NOTE`` segment.
+When using a split core file layout, this note is in the AMDGPU file.
+
+The note record vendor field is "``AMDGPU``" and the record type is
+"``NT_AMDGPU_KFD_CORE_STATE``" (see :ref:`amdgpu-note-records-v3-onwards`)
+
+The content of the note is defined in table
+:ref:`amdgpu-core-snapshot-note-layout-table-v1`:
+
+  .. table:: AMDGPU snapshot note format V1
+     :name: amdgpu-core-snapshot-note-layout-table-v1
+
+     ================================ ======================================= ======================= ============== ===========================
+     Field                            Type                                    Size (bytes)            Byte alignment Comment
+     ================================ ======================================= ======================= ============== ===========================
+     ``version_major``                ``uint32``                              4                       4              ``KFD_IOCTL_MAJOR_VERSION``
+     ``version_minor``                ``uint32``                              4                       4              ``KFD_IOCTL_MINOR_VERSION``
+     ``runtime_info_size``            ``uint64``                              8                       8              Must be a multiple of 8
+     ``n_agents``                     ``uint32``                              4                       8
+     ``agent_info_entry_size``        ``uint32``                              4                       4              Must be a multiple of 8
+     ``n_queues``                     ``uint32``                              4                       8
+     ``queue_info_entry_size``        ``uint32``                              4                       4              Must be a multiple of 8
+     ``runtime_info``                 ``kfd_runtime_info``                    ``runtime_info_size``   8
+     ``agents_info``                  ``kfd_dbg_device_info_entry[n_agents]`` ``n_agents *            8
+                                                                              agent_info_entry_size``
+     ``queues_info``                  ``kfd_queue_snapshot_entry[n_queues]``  ``n_queues *
+                                                                              queue_info_entry_size`` 8
+     ================================ ======================================= ======================= ============== ===========================
+
+The definition of all the ``kfd_*`` types comes from the
+``include/uapi/linux/kfd_ioctl.h`` header file from the KFD repository. It is
+usually installed in ``/usr/include/linux/kfd_ioctl.h``. The version of the
+``kfd_ioctl.h`` file used must define values for
+``KFD_IOCTL_MAJOR_VERSION`` and ``KFD_IOCTL_MINOR_VERSION`` matching
+the values of ``kfd_version_major`` and ``kfd_version_major`` from the
+note.
+
+.. _amdgpu_corefile_memory:
+
+Memory segments
+---------------
+
+An AMDGPU core file must contain an image of the AMDGPU agents' memory in load
+segments (of type ``PT_LOAD``).  Those segments must correspond to the memory
+regions where the content of the agent memory is mapped into the host process
+by the ROCr runtime (note that those memory mappings are usually not readable
+by the process itself).
+
+When using the split core file layout, those segments must be included in the
+AMDGPU core file.
+
 Source Languages
 ================
 
diff --git a/llvm/docs/CodeReview.rst b/llvm/docs/CodeReview.rst
index ca57cffc519de8a..0128e37465de321 100644
--- a/llvm/docs/CodeReview.rst
+++ b/llvm/docs/CodeReview.rst
@@ -125,6 +125,11 @@ commit message).
 If you suggest changes in a code review, but don't wish the suggestion to be
 interpreted this strongly, please state so explicitly.
 
+.. note::
+   After responding to reviewer comments,
+   press `Re-request review <https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review#:~:text=After%20your%20pull%20request%20is%20reviewed>`_
+   to bring the Pull Request to the reviewers' attention.
+
 Aim to Make Efficient Use of Everyone's Time
 --------------------------------------------
 
diff --git a/llvm/docs/DirectX/DXILOpTableGenDesign.rst b/llvm/docs/DirectX/DXILOpTableGenDesign.rst
index 50d801bd05efdbc..46106ee2a50f572 100644
--- a/llvm/docs/DirectX/DXILOpTableGenDesign.rst
+++ b/llvm/docs/DirectX/DXILOpTableGenDesign.rst
@@ -93,18 +93,14 @@ properties are specified as fields of the ``DXILOp`` class as described below.
         class DXILOpClass;
 
    Concrete operation records, such as ``unary`` are defined by inheriting from ``DXILOpClass``.
-6. Return type of the operation is represented as ``LLVMType``.
-7. Operation arguments are represented as a list of ``LLVMType`` with each type
-   corresponding to the argument position. An overload type, if supported by the operation, is
-   denoted as the positional type ``overloadTy`` in the argument or in the result, where
-   ``overloadTy`` is defined to be synonymous to ``llvm_any_ty``.
-
-   .. code-block::
-
-      defvar overloadTy = llvm_any_ty
-
-   Empty list, ``[]`` represents an operation with no arguments.
-
+6. A set of type names are defined that represent return and argument types,
+   which all inherit from ``DXILOpParamType``. These represent simple types
+   like ``int32Ty``, DXIL types like ``dx.types.Handle``, and a special
+   ``overloadTy`` which can be any type allowed by ``Overloads``, described
+   below.
+7. Operation return type is represented as a ``DXILOpParamType``, and arguments
+   are represented as a list of the same. An operation with no return value
+   shall specify ``VoidTy`` as its return.
 8. Valid operation overload types predicated on DXIL version are specified as
    a list of ``Overloads`` records. Representation of ``Overloads``
    class is described in a later section.
@@ -145,10 +141,10 @@ TableGen representations of its properties described above.
      Intrinsic LLVMIntrinsic = ?;
 
      // Result type of the op.
-     LLVMType result;
+     DXILOpParamType result;
 
      // List of argument types of the op. Default to 0 arguments.
-     list<LLVMType> arguments = [];
+     list<DXILOpParamType> arguments = [];
 
      // List of valid overload types predicated by DXIL version
      list<Overloads> overloads;
@@ -233,9 +229,9 @@ overloads predicated on DXIL version as list of records of the following class
 
 .. code-block::
 
-   class Overloads<Version minver, list<LLVMType> ols> {
+   class Overloads<Version minver, list<DXILOpParamType> ols> {
      Version dxil_version = minver;
-     list<LLVMType> overload_types = ols;
+     list<DXILOpParamType> overload_types = ols;
    }
 
 Following is an example specification of valid overload types for ``DXIL1_0`` and
diff --git a/llvm/docs/ExtendingLLVM.rst b/llvm/docs/ExtendingLLVM.rst
index 12372996c96dde1..3965112a20f9080 100644
--- a/llvm/docs/ExtendingLLVM.rst
+++ b/llvm/docs/ExtendingLLVM.rst
@@ -90,14 +90,14 @@ complicated behavior in a single node (rotate).
 
    Add an enum value for the new SelectionDAG node.
 
-#. ``lib/CodeGen/SelectionDAG/SelectionDAG.cpp``:
+#. ``lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp``:
 
    Add code to print the node to ``getOperationName``.  If your new node can be
-    evaluated at compile time when given constant arguments (such as an add of a
-    constant with another constant), find the ``getNode`` method that takes the
-    appropriate number of arguments, and add a case for your node to the switch
-    statement that performs constant folding for nodes that take the same number
-    of arguments as your new node.
+   evaluated at compile time when given constant arguments (such as an add of a
+   constant with another constant), find the ``getNode`` method that takes the
+   appropriate number of arguments, and add a case for your node to the switch
+   statement that performs constant folding for nodes that take the same number
+   of arguments as your new node.
 
 #. ``lib/CodeGen/SelectionDAG/LegalizeDAG.cpp``:
 
@@ -115,12 +115,12 @@ complicated behavior in a single node (rotate).
 #. ``lib/CodeGen/SelectionDAG/LegalizeDAG.cpp``:
 
    If targets may support the new node being added only at certain sizes, you
-    will also need to add code to your node's case statement in ``LegalizeOp``
-    to Promote your node's operands to a larger size, and perform the correct
-    operation.  You will also need to add code to ``PromoteOp`` to do this as
-    well.  For a good example, see ``ISD::BSWAP``, which promotes its operand to
-    a wider size, performs the byteswap, and then shifts the correct bytes right
-    to emulate the narrower byteswap in the wider type.
+   will also need to add code to your node's case statement in ``LegalizeOp``
+   to Promote your node's operands to a larger size, and perform the correct
+   operation.  You will also need to add code to ``PromoteOp`` to do this as
+   well.  For a good example, see ``ISD::BSWAP``, which promotes its operand to
+   a wider size, performs the byteswap, and then shifts the correct bytes right
+   to emulate the narrower byteswap in the wider type.
 
 #. ``lib/CodeGen/SelectionDAG/LegalizeDAG.cpp``:
 
diff --git a/llvm/docs/GitHub.rst b/llvm/docs/GitHub.rst
index 14424f722abf63d..a33211bfef41def 100644
--- a/llvm/docs/GitHub.rst
+++ b/llvm/docs/GitHub.rst
@@ -23,6 +23,23 @@ intended to be able to support "stacked" pull-request. Do not create any branche
 llvm/llvm-project repository otherwise, please use a fork (see below). User branches that
 aren't associated with a pull-request **will be deleted**.
 
+Using Graphite for stacked Pull Requests
+========================================
+
+[Graphite](https://app.graphite.dev/) is a stacked pull request tool supported
+by the LLVM repo (the other being [reviewable.io](https://reviewable.io)).
+
+Graphite will want to create branches under `llvm/llvm-project` rather than your
+private fork, so the guidance above, about branch naming, is critical, otherwise
+`gt submit` (i.e. publish your PRs for review) will fail.
+
+Use `gt config` then `Branch naming settings` and `Set a prefix for branch names`.
+Include the last `/`.
+
+If you didn't do the above and Graphite created non-prefixed branches, a simple way to
+unblock is to rename (`git -m <old name> <new name>`), and then checkout the branch
+and `gt track`.
+
 Pull Requests
 =============
 The LLVM project is using GitHub Pull Requests for Code Reviews. This document
diff --git a/llvm/docs/PointerAuth.md b/llvm/docs/PointerAuth.md
index e027c902e58e123..84e0af7577c7d6d 100644
--- a/llvm/docs/PointerAuth.md
+++ b/llvm/docs/PointerAuth.md
@@ -374,6 +374,19 @@ For example:
     .quad _sym@AUTH(ia,12,addr)
 ```
 
+#### MachO Object File Representation
+
+At the object file level, authenticated relocations are represented using the
+``ARM64_RELOC_AUTHENTICATED_POINTER`` relocation kind (with value ``11``).
+
+The pointer authentication information is encoded into the addend as follows:
+
+```
+| 63 | 62 | 61-51 | 50-49 |   48   | 47     -     32 | 31  -  0 |
+| -- | -- | ----- | ----- | ------ | --------------- | -------- |
+|  1 |  0 |   0   |  key  |  addr  |  discriminator  |  addend  |
+```
+
 #### ELF Object File Representation
 
 At the object file level, authenticated relocations are represented
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index b3c7b0e3883d01f..3af1428138c0216 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -302,6 +302,9 @@ The primary goal of experimental support is to assist in the process of ratifica
 ``experimental-zicfilp``, ``experimental-zicfiss``
   LLVM implements the `1.0 release specification <https://github.com/riscv/riscv-cfi/releases/tag/v1.0>`__.
 
+``experimental-zvbc32e``, ``experimental-zvkgs``
+  LLVM implements the `0.7 release specification <https://github.com/user-attachments/files/16450464/riscv-crypto-spec-vector-extra_v0.0.7.pdf>`__.
+
 To use an experimental extension from `clang`, you must add `-menable-experimental-extensions` to the command line, and specify the exact version of the experimental extension you are using.  To use an experimental extension with LLVM's internal developer tools (e.g. `llc`, `llvm-objdump`, `llvm-mc`), you must prefix the extension name with `experimental-`.  Note that you don't need to specify the version with internal tools, and shouldn't include the `experimental-` prefix with `clang`.
 
 Vendor Extensions
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 67175e13a8e9ffa..d40bb2682f9ad82 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -111,6 +111,7 @@ Changes to the RISC-V Backend
 * Fixed length vector support using RVV instructions now requires VLEN>=64. This
   means Zve32x and Zve32f will also require Zvl64b. The prior support was
   largely untested.
+* The ``Zvbc32e`` and ``Zvkgs`` extensions are now supported experimentally.
 
 Changes to the WebAssembly Backend
 ----------------------------------
@@ -151,6 +152,17 @@ Changes to the C API
   * ``LLVMX86MMXTypeInContext``
   * ``LLVMX86MMXType``
 
+ * The following functions are added to further support non-null-terminated strings:
+
+  * ``LLVMGetNamedFunctionWithLength``
+  * ``LLVMGetNamedGlobalWithLength``
+
+* The new pass manager can now be invoked with a custom alias analysis pipeline, using
+  the ``LLVMPassBuilderOptionsSetAAPipeline`` function.
+
+* It is now also possible to run the new pass manager on a single function, by calling
+  ``LLVMRunPassesOnFunction`` instead of ``LLVMRunPasses``.
+
 Changes to the CodeGen infrastructure
 -------------------------------------
 
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 4bde82d816e35cf..7d2e7c955207613 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -1179,6 +1179,16 @@ LLVMValueRef LLVMAddFunction(LLVMModuleRef M, const char *Name,
  */
 LLVMValueRef LLVMGetNamedFunction(LLVMModuleRef M, const char *Name);
 
+/**
+ * Obtain a Function value from a Module by its name.
+ *
+ * The returned value corresponds to a llvm::Function value.
+ *
+ * @see llvm::Module::getFunction()
+ */
+LLVMValueRef LLVMGetNamedFunctionWithLength(LLVMModuleRef M, const char *Name,
+                                            size_t Length);
+
 /**
  * Obtain an iterator to the first Function in a Module.
  *
@@ -2633,6 +2643,8 @@ LLVMValueRef LLVMAddGlobalInAddressSpace(LLVMModuleRef M, LLVMTypeRef Ty,
                                          const char *Name,
                                          unsigned AddressSpace);
 LLVMValueRef LLVMGetNamedGlobal(LLVMModuleRef M, const char *Name);
+LLVMValueRef LLVMGetNamedGlobalWithLength(LLVMModuleRef M, const char *Name,
+                                          size_t Length);
 LLVMValueRef LLVMGetFirstGlobal(LLVMModuleRef M);
 LLVMValueRef LLVMGetLastGlobal(LLVMModuleRef M);
 LLVMValueRef LLVMGetNextGlobal(LLVMValueRef GlobalVar);
diff --git a/llvm/include/llvm-c/Transforms/PassBuilder.h b/llvm/include/llvm-c/Transforms/PassBuilder.h
index 03a5abaa753312a..d297b57cadd07d3 100644
--- a/llvm/include/llvm-c/Transforms/PassBuilder.h
+++ b/llvm/include/llvm-c/Transforms/PassBuilder.h
@@ -50,6 +50,16 @@ LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
                            LLVMTargetMachineRef TM,
                            LLVMPassBuilderOptionsRef Options);
 
+/**
+ * Construct and run a set of passes over a function.
+ *
+ * This function behaves the same as LLVMRunPasses, but operates on a single
+ * function instead of an entire module.
+ */
+LLVMErrorRef LLVMRunPassesOnFunction(LLVMValueRef F, const char *Passes,
+                                     LLVMTargetMachineRef TM,
+                                     LLVMPassBuilderOptionsRef Options);
+
 /**
  * Create a new set of options for a PassBuilder
  *
diff --git a/llvm/include/llvm/ADT/APFixedPoint.h b/llvm/include/llvm/ADT/APFixedPoint.h
index 0c014e76aa71260..ae40db96e4818c7 100644
--- a/llvm/include/llvm/ADT/APFixedPoint.h
+++ b/llvm/include/llvm/ADT/APFixedPoint.h
@@ -17,7 +17,6 @@
 #define LLVM_ADT_APFIXEDPOINT_H
 
 #include "llvm/ADT/APSInt.h"
-#include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 108df7e0eaeaa33..65ba3f15305c78f 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -79,13 +79,11 @@ class [[nodiscard]] APInt {
 public:
   typedef uint64_t WordType;
 
-  /// This enum is used to hold the constants we needed for APInt.
-  enum : unsigned {
-    /// Byte size of a word.
-    APINT_WORD_SIZE = sizeof(WordType),
-    /// Bits in a word.
-    APINT_BITS_PER_WORD = APINT_WORD_SIZE * CHAR_BIT
-  };
+  /// Byte size of a word.
+  static constexpr unsigned APINT_WORD_SIZE = sizeof(WordType);
+
+  /// Bits in a word.
+  static constexpr unsigned APINT_BITS_PER_WORD = APINT_WORD_SIZE * CHAR_BIT;
 
   enum class Rounding {
     DOWN,
diff --git a/llvm/include/llvm/ADT/DeltaTree.h b/llvm/include/llvm/ADT/DeltaTree.h
new file mode 100644
index 000000000000000..5db9d74d9d5296f
--- /dev/null
+++ b/llvm/include/llvm/ADT/DeltaTree.h
@@ -0,0 +1,50 @@
+//===- DeltaTree.h - B-Tree for Rewrite Delta tracking ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the DeltaTree class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_DELTATREE_H
+#define LLVM_ADT_DELTATREE_H
+
+namespace llvm {
+
+/// DeltaTree - a multiway search tree (BTree) structure with some fancy
+/// features.  B-Trees are generally more memory and cache efficient than
+/// binary trees, because they store multiple keys/values in each node.  This
+/// implements a key/value mapping from index to delta, and allows fast lookup
+/// on index.  However, an added (important) bonus is that it can also
+/// efficiently tell us the full accumulated delta for a specific file offset
+/// as well, without traversing the whole tree.
+class DeltaTree {
+  void *Root; // "DeltaTreeNode *"
+
+public:
+  DeltaTree();
+
+  // Note: Currently we only support copying when the RHS is empty.
+  DeltaTree(const DeltaTree &RHS);
+
+  DeltaTree &operator=(const DeltaTree &) = delete;
+  ~DeltaTree();
+
+  /// getDeltaAt - Return the accumulated delta at the specified file offset.
+  /// This includes all insertions or delections that occurred *before* the
+  /// specified file index.
+  int getDeltaAt(unsigned FileIndex) const;
+
+  /// AddDelta - When a change is made that shifts around the text buffer,
+  /// this method is used to record that info.  It inserts a delta of 'Delta'
+  /// into the current DeltaTree at offset FileIndex.
+  void AddDelta(unsigned FileIndex, int Delta);
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_DELTATREE_H
diff --git a/llvm/include/llvm/ADT/DynamicAPInt.h b/llvm/include/llvm/ADT/DynamicAPInt.h
index 4444d52527def13..ff958d48e773179 100644
--- a/llvm/include/llvm/ADT/DynamicAPInt.h
+++ b/llvm/include/llvm/ADT/DynamicAPInt.h
@@ -18,10 +18,12 @@
 
 #include "llvm/ADT/SlowDynamicAPInt.h"
 #include "llvm/Support/MathExtras.h"
-#include "llvm/Support/raw_ostream.h"
 #include <numeric>
 
 namespace llvm {
+
+class raw_ostream;
+
 /// This class provides support for dynamic arbitrary-precision arithmetic.
 ///
 /// Unlike APInt, this extends the precision as necessary to prevent overflows
diff --git a/llvm/include/llvm/ADT/FunctionExtras.h b/llvm/include/llvm/ADT/FunctionExtras.h
index 49e0e8ab0db400d..d92868e3715f40e 100644
--- a/llvm/include/llvm/ADT/FunctionExtras.h
+++ b/llvm/include/llvm/ADT/FunctionExtras.h
@@ -80,6 +80,7 @@ using EnableIfCallable = std::enable_if_t<std::disjunction<
 template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
 protected:
   static constexpr size_t InlineStorageSize = sizeof(void *) * 3;
+  static constexpr size_t InlineStorageAlign = alignof(void *);
 
   template <typename T, class = void>
   struct IsSizeLessThanThresholdT : std::false_type {};
@@ -161,7 +162,8 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
     // provide three pointers worth of storage here.
     // This is mutable as an inlined `const unique_function<void() const>` may
     // still modify its own mutable members.
-    alignas(void *) mutable std::byte InlineStorage[InlineStorageSize];
+    alignas(InlineStorageAlign) mutable std::byte
+        InlineStorage[InlineStorageSize];
   } StorageUnion;
 
   // A compressed pointer to either our dispatching callback or our table of
@@ -262,7 +264,7 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
     bool IsInlineStorage = true;
     void *CallableAddr = getInlineStorage();
     if (sizeof(CallableT) > InlineStorageSize ||
-        alignof(CallableT) > alignof(decltype(StorageUnion.InlineStorage))) {
+        alignof(CallableT) > InlineStorageAlign) {
       IsInlineStorage = false;
       // Allocate out-of-line storage. FIXME: Use an explicit alignment
       // parameter in C++17 mode.
@@ -312,6 +314,7 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
       // Non-trivial move, so dispatch to a type-erased implementation.
       getNonTrivialCallbacks()->MovePtr(getInlineStorage(),
                                         RHS.getInlineStorage());
+      getNonTrivialCallbacks()->DestroyPtr(RHS.getInlineStorage());
     }
 
     // Clear the old callback and inline flag to get back to as-if-null.
diff --git a/clang/include/clang/Rewrite/Core/RewriteBuffer.h b/llvm/include/llvm/ADT/RewriteBuffer.h
similarity index 82%
rename from clang/include/clang/Rewrite/Core/RewriteBuffer.h
rename to llvm/include/llvm/ADT/RewriteBuffer.h
index b8f34174b715664..8197a06f882c1aa 100644
--- a/clang/include/clang/Rewrite/Core/RewriteBuffer.h
+++ b/llvm/include/llvm/ADT/RewriteBuffer.h
@@ -6,15 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_REWRITE_CORE_REWRITEBUFFER_H
-#define LLVM_CLANG_REWRITE_CORE_REWRITEBUFFER_H
+#ifndef LLVM_ADT_REWRITEBUFFER_H
+#define LLVM_ADT_REWRITEBUFFER_H
 
-#include "clang/Basic/LLVM.h"
-#include "clang/Rewrite/Core/DeltaTree.h"
-#include "clang/Rewrite/Core/RewriteRope.h"
+#include "llvm/ADT/DeltaTree.h"
+#include "llvm/ADT/RewriteRope.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace clang {
+class Rewriter;
+} // namespace clang
+
+namespace llvm {
+
+class raw_ostream;
 
 /// RewriteBuffer - As code is rewritten, SourceBuffer's from the original
 /// input with modifications get a new RewriteBuffer associated with them.  The
@@ -23,7 +28,7 @@ namespace clang {
 /// RewriteBuffer.  For example, if text is inserted into the buffer, any
 /// locations after the insertion point have to be mapped.
 class RewriteBuffer {
-  friend class Rewriter;
+  friend class clang::Rewriter;
 
   /// Deltas - Keep track of all the deltas in the source code due to insertions
   /// and deletions.
@@ -43,9 +48,7 @@ class RewriteBuffer {
   void Initialize(const char *BufStart, const char *BufEnd) {
     Buffer.assign(BufStart, BufEnd);
   }
-  void Initialize(StringRef Input) {
-    Initialize(Input.begin(), Input.end());
-  }
+  void Initialize(StringRef Input) { Initialize(Input.begin(), Input.end()); }
 
   /// Write to \p Stream the result of applying all changes to the
   /// original buffer.
@@ -63,9 +66,7 @@ class RewriteBuffer {
   /// InsertText - Insert some text at the specified point, where the offset in
   /// the buffer is specified relative to the original SourceBuffer.  The
   /// text is inserted after the specified location.
-  void InsertText(unsigned OrigOffset, StringRef Str,
-                  bool InsertAfter = true);
-
+  void InsertText(unsigned OrigOffset, StringRef Str, bool InsertAfter = true);
 
   /// InsertTextBefore - Insert some text before the specified point, where the
   /// offset in the buffer is specified relative to the original
@@ -85,8 +86,7 @@ class RewriteBuffer {
   /// ReplaceText - This method replaces a range of characters in the input
   /// buffer with a new string.  This is effectively a combined "remove/insert"
   /// operation.
-  void ReplaceText(unsigned OrigOffset, unsigned OrigLength,
-                   StringRef NewStr);
+  void ReplaceText(unsigned OrigOffset, unsigned OrigLength, StringRef NewStr);
 
 private:
   /// getMappedOffset - Given an offset into the original SourceBuffer that this
@@ -95,23 +95,23 @@ class RewriteBuffer {
   /// position where text is inserted, the location returned will be after any
   /// inserted text at the position.
   unsigned getMappedOffset(unsigned OrigOffset,
-                           bool AfterInserts = false) const{
-    return Deltas.getDeltaAt(2*OrigOffset+AfterInserts)+OrigOffset;
+                           bool AfterInserts = false) const {
+    return Deltas.getDeltaAt(2 * OrigOffset + AfterInserts) + OrigOffset;
   }
 
   /// AddInsertDelta - When an insertion is made at a position, this
   /// method is used to record that information.
   void AddInsertDelta(unsigned OrigOffset, int Change) {
-    return Deltas.AddDelta(2*OrigOffset, Change);
+    return Deltas.AddDelta(2 * OrigOffset, Change);
   }
 
   /// AddReplaceDelta - When a replacement/deletion is made at a position, this
   /// method is used to record that information.
   void AddReplaceDelta(unsigned OrigOffset, int Change) {
-    return Deltas.AddDelta(2*OrigOffset+1, Change);
+    return Deltas.AddDelta(2 * OrigOffset + 1, Change);
   }
 };
 
-} // namespace clang
+} // namespace llvm
 
-#endif // LLVM_CLANG_REWRITE_CORE_REWRITEBUFFER_H
+#endif // LLVM_ADT_REWRITEBUFFER_H
diff --git a/llvm/include/llvm/ADT/RewriteRope.h b/llvm/include/llvm/ADT/RewriteRope.h
new file mode 100644
index 000000000000000..784f3c07eaa62bc
--- /dev/null
+++ b/llvm/include/llvm/ADT/RewriteRope.h
@@ -0,0 +1,223 @@
+//===- RewriteRope.h - Rope specialized for rewriter ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the RewriteRope class, which is a powerful string class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_REWRITEROPE_H
+#define LLVM_ADT_REWRITEROPE_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/StringRef.h"
+#include <cassert>
+#include <cstddef>
+#include <iterator>
+#include <utility>
+
+namespace llvm {
+
+//===--------------------------------------------------------------------===//
+// RopeRefCountString Class
+//===--------------------------------------------------------------------===//
+
+/// RopeRefCountString - This struct is allocated with 'new char[]' from the
+/// heap, and represents a reference counted chunk of string data.  When its
+/// ref count drops to zero, it is delete[]'d.  This is primarily managed
+/// through the RopePiece class below.
+struct RopeRefCountString {
+  unsigned RefCount;
+  char Data[1]; //  Variable sized.
+
+  void Retain() { ++RefCount; }
+
+  void Release() {
+    assert(RefCount > 0 && "Reference count is already zero.");
+    if (--RefCount == 0)
+      delete[] (char *)this;
+  }
+};
+
+//===--------------------------------------------------------------------===//
+// RopePiece Class
+//===--------------------------------------------------------------------===//
+
+/// RopePiece - This class represents a view into a RopeRefCountString object.
+/// This allows references to string data to be efficiently chopped up and
+/// moved around without having to push around the string data itself.
+///
+/// For example, we could have a 1M RopePiece and want to insert something
+/// into the middle of it.  To do this, we split it into two RopePiece objects
+/// that both refer to the same underlying RopeRefCountString (just with
+/// different offsets) which is a nice constant time operation.
+struct RopePiece {
+  llvm::IntrusiveRefCntPtr<RopeRefCountString> StrData;
+  unsigned StartOffs = 0;
+  unsigned EndOffs = 0;
+
+  RopePiece() = default;
+  RopePiece(llvm::IntrusiveRefCntPtr<RopeRefCountString> Str, unsigned Start,
+            unsigned End)
+      : StrData(std::move(Str)), StartOffs(Start), EndOffs(End) {}
+
+  const char &operator[](unsigned Offset) const {
+    return StrData->Data[Offset + StartOffs];
+  }
+  char &operator[](unsigned Offset) {
+    return StrData->Data[Offset + StartOffs];
+  }
+
+  unsigned size() const { return EndOffs - StartOffs; }
+};
+
+//===--------------------------------------------------------------------===//
+// RopePieceBTreeIterator Class
+//===--------------------------------------------------------------------===//
+
+/// RopePieceBTreeIterator - This class provides read-only forward iteration
+/// over bytes that are in a RopePieceBTree.  This first iterates over bytes
+/// in a RopePiece, then iterates over RopePiece's in a RopePieceBTreeLeaf,
+/// then iterates over RopePieceBTreeLeaf's in a RopePieceBTree.
+class RopePieceBTreeIterator {
+  /// CurNode - The current B+Tree node that we are inspecting.
+  const void /*RopePieceBTreeLeaf*/ *CurNode = nullptr;
+
+  /// CurPiece - The current RopePiece in the B+Tree node that we're
+  /// inspecting.
+  const RopePiece *CurPiece = nullptr;
+
+  /// CurChar - The current byte in the RopePiece we are pointing to.
+  unsigned CurChar = 0;
+
+public:
+  using iterator_category = std::forward_iterator_tag;
+  using value_type = const char;
+  using difference_type = std::ptrdiff_t;
+  using pointer = value_type *;
+  using reference = value_type &;
+
+  RopePieceBTreeIterator() = default;
+  RopePieceBTreeIterator(const void /*RopePieceBTreeNode*/ *N);
+
+  char operator*() const { return (*CurPiece)[CurChar]; }
+
+  bool operator==(const RopePieceBTreeIterator &RHS) const {
+    return CurPiece == RHS.CurPiece && CurChar == RHS.CurChar;
+  }
+  bool operator!=(const RopePieceBTreeIterator &RHS) const {
+    return !operator==(RHS);
+  }
+
+  RopePieceBTreeIterator &operator++() { // Preincrement
+    if (CurChar + 1 < CurPiece->size())
+      ++CurChar;
+    else
+      MoveToNextPiece();
+    return *this;
+  }
+
+  RopePieceBTreeIterator operator++(int) { // Postincrement
+    RopePieceBTreeIterator tmp = *this;
+    ++*this;
+    return tmp;
+  }
+
+  llvm::StringRef piece() const {
+    return llvm::StringRef(&(*CurPiece)[0], CurPiece->size());
+  }
+
+  void MoveToNextPiece();
+};
+
+//===--------------------------------------------------------------------===//
+// RopePieceBTree Class
+//===--------------------------------------------------------------------===//
+
+class RopePieceBTree {
+  void /*RopePieceBTreeNode*/ *Root;
+
+public:
+  RopePieceBTree();
+  RopePieceBTree(const RopePieceBTree &RHS);
+  RopePieceBTree &operator=(const RopePieceBTree &) = delete;
+  ~RopePieceBTree();
+
+  using iterator = RopePieceBTreeIterator;
+
+  iterator begin() const { return iterator(Root); }
+  iterator end() const { return iterator(); }
+  unsigned size() const;
+  unsigned empty() const { return size() == 0; }
+
+  void clear();
+
+  void insert(unsigned Offset, const RopePiece &R);
+
+  void erase(unsigned Offset, unsigned NumBytes);
+};
+
+//===--------------------------------------------------------------------===//
+// RewriteRope Class
+//===--------------------------------------------------------------------===//
+
+/// RewriteRope - A powerful string class.  This class supports extremely
+/// efficient insertions and deletions into the middle of it, even for
+/// ridiculously long strings.
+class RewriteRope {
+  RopePieceBTree Chunks;
+
+  /// We allocate space for string data out of a buffer of size AllocChunkSize.
+  /// This keeps track of how much space is left.
+  llvm::IntrusiveRefCntPtr<RopeRefCountString> AllocBuffer;
+  enum { AllocChunkSize = 4080 };
+  unsigned AllocOffs = AllocChunkSize;
+
+public:
+  RewriteRope() = default;
+  RewriteRope(const RewriteRope &RHS) : Chunks(RHS.Chunks) {}
+
+  // The copy assignment operator is defined as deleted pending further
+  // motivation.
+  RewriteRope &operator=(const RewriteRope &) = delete;
+
+  using iterator = RopePieceBTree::iterator;
+  using const_iterator = RopePieceBTree::iterator;
+
+  iterator begin() const { return Chunks.begin(); }
+  iterator end() const { return Chunks.end(); }
+  unsigned size() const { return Chunks.size(); }
+
+  void clear() { Chunks.clear(); }
+
+  void assign(const char *Start, const char *End) {
+    clear();
+    if (Start != End)
+      Chunks.insert(0, MakeRopeString(Start, End));
+  }
+
+  void insert(unsigned Offset, const char *Start, const char *End) {
+    assert(Offset <= size() && "Invalid position to insert!");
+    if (Start == End)
+      return;
+    Chunks.insert(Offset, MakeRopeString(Start, End));
+  }
+
+  void erase(unsigned Offset, unsigned NumBytes) {
+    assert(Offset + NumBytes <= size() && "Invalid region to erase!");
+    if (NumBytes == 0)
+      return;
+    Chunks.erase(Offset, NumBytes);
+  }
+
+private:
+  RopePiece MakeRopeString(const char *Start, const char *End);
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_REWRITEROPE_H
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index 3f5ea3a5275f35b..b2df0cd650ad7b3 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -1843,6 +1843,13 @@ OutputIt replace_copy(R &&Range, OutputIt Out, const T &OldValue,
                            NewValue);
 }
 
+/// Provide wrappers to std::replace which take ranges instead of having to pass
+/// begin/end explicitly.
+template <typename R, typename T>
+void replace(R &&Range, const T &OldValue, const T &NewValue) {
+  std::replace(adl_begin(Range), adl_end(Range), OldValue, NewValue);
+}
+
 /// Provide wrappers to std::move which take ranges instead of having to
 /// pass begin/end explicitly.
 template <typename R, typename OutputIt>
@@ -1975,6 +1982,8 @@ auto upper_bound(R &&Range, T &&Value, Compare C) {
                           std::forward<T>(Value), C);
 }
 
+/// Provide wrappers to std::min_element which take ranges instead of having to
+/// pass begin/end explicitly.
 template <typename R> auto min_element(R &&Range) {
   return std::min_element(adl_begin(Range), adl_end(Range));
 }
@@ -1983,6 +1992,8 @@ template <typename R, typename Compare> auto min_element(R &&Range, Compare C) {
   return std::min_element(adl_begin(Range), adl_end(Range), C);
 }
 
+/// Provide wrappers to std::max_element which take ranges instead of having to
+/// pass begin/end explicitly.
 template <typename R> auto max_element(R &&Range) {
   return std::max_element(adl_begin(Range), adl_end(Range));
 }
@@ -1991,6 +2002,20 @@ template <typename R, typename Compare> auto max_element(R &&Range, Compare C) {
   return std::max_element(adl_begin(Range), adl_end(Range), C);
 }
 
+/// Provide wrappers to std::mismatch which take ranges instead of having to
+/// pass begin/end explicitly.
+/// This function returns a pair of iterators for the first mismatching elements
+/// from `R1` and `R2`. As an example, if:
+///
+/// R1 = [0, 1, 4, 6], R2 = [0, 1, 5, 6]
+///
+/// this function will return a pair of iterators, first pointing to R1[2] and
+/// second pointing to R2[2].
+template <typename R1, typename R2> auto mismatch(R1 &&Range1, R2 &&Range2) {
+  return std::mismatch(adl_begin(Range1), adl_end(Range1), adl_begin(Range2),
+                       adl_end(Range2));
+}
+
 template <typename R>
 void stable_sort(R &&Range) {
   std::stable_sort(adl_begin(Range), adl_end(Range));
diff --git a/llvm/include/llvm/ADT/SlowDynamicAPInt.h b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
index cda5f39eb98c359..ec1021892cf4ddd 100644
--- a/llvm/include/llvm/ADT/SlowDynamicAPInt.h
+++ b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
@@ -19,10 +19,10 @@
 #define LLVM_ADT_SLOWDYNAMICAPINT_H
 
 #include "llvm/ADT/APInt.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 class DynamicAPInt;
+class raw_ostream;
 } // namespace llvm
 
 namespace llvm::detail {
diff --git a/llvm/include/llvm/ADT/SmallPtrSet.h b/llvm/include/llvm/ADT/SmallPtrSet.h
index 78383f495b6a58d..1fc2318342ae784 100644
--- a/llvm/include/llvm/ADT/SmallPtrSet.h
+++ b/llvm/include/llvm/ADT/SmallPtrSet.h
@@ -16,7 +16,6 @@
 #define LLVM_ADT_SMALLPTRSET_H
 
 #include "llvm/ADT/EpochTracker.h"
-#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ReverseIteration.h"
 #include "llvm/Support/type_traits.h"
@@ -186,8 +185,8 @@ class SmallPtrSetImplBase : public DebugEpochBase {
       return false;
     }
 
-    auto *Bucket = FindBucketFor(Ptr);
-    if (*Bucket != Ptr)
+    auto *Bucket = doFind(Ptr);
+    if (!Bucket)
       return false;
 
     *const_cast<const void **>(Bucket) = getTombstoneMarker();
@@ -212,8 +211,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
     }
 
     // Big set case.
-    auto *Bucket = FindBucketFor(Ptr);
-    if (*Bucket == Ptr)
+    if (auto *Bucket = doFind(Ptr))
       return Bucket;
     return EndPointer();
   }
@@ -223,6 +221,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
 private:
   std::pair<const void *const *, bool> insert_imp_big(const void *Ptr);
 
+  const void *const *doFind(const void *Ptr) const;
   const void * const *FindBucketFor(const void *Ptr) const;
   void shrink_and_clear();
 
diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h
index 17444147b102a96..730f84ca038d60c 100644
--- a/llvm/include/llvm/ADT/SmallVector.h
+++ b/llvm/include/llvm/ADT/SmallVector.h
@@ -15,7 +15,6 @@
 #define LLVM_ADT_SMALLVECTOR_H
 
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/type_traits.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
diff --git a/llvm/include/llvm/ADT/Statistic.h b/llvm/include/llvm/ADT/Statistic.h
index 9475b6697031a46..a64cfce3ba44a0c 100644
--- a/llvm/include/llvm/ADT/Statistic.h
+++ b/llvm/include/llvm/ADT/Statistic.h
@@ -27,7 +27,6 @@
 #define LLVM_ADT_STATISTIC_H
 
 #include "llvm/Config/llvm-config.h"
-#include "llvm/Support/Compiler.h"
 #include <atomic>
 #include <memory>
 #include <vector>
diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index 4140387a1f34109..1990172116582b2 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -38,7 +38,6 @@
 #define LLVM_ANALYSIS_ALIASANALYSIS_H
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/IR/Function.h"
@@ -53,18 +52,14 @@
 
 namespace llvm {
 
-class AnalysisUsage;
 class AtomicCmpXchgInst;
 class BasicBlock;
 class CatchPadInst;
 class CatchReturnInst;
 class DominatorTree;
 class FenceInst;
-class Function;
 class LoopInfo;
-class PreservedAnalyses;
 class TargetLibraryInfo;
-class Value;
 
 /// The possible results of an alias query.
 ///
diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h
index b8e8e7ef047f963..e5817d2409bc651 100644
--- a/llvm/include/llvm/Analysis/AliasSetTracker.h
+++ b/llvm/include/llvm/Analysis/AliasSetTracker.h
@@ -23,7 +23,6 @@
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
 #include "llvm/Analysis/MemoryLocation.h"
-#include "llvm/IR/Instruction.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
 #include <cassert>
@@ -37,10 +36,12 @@ class AnyMemSetInst;
 class AnyMemTransferInst;
 class BasicBlock;
 class BatchAAResults;
+class Function;
+class Instruction;
+class StoreInst;
 class LoadInst;
 enum class ModRefInfo : uint8_t;
 class raw_ostream;
-class StoreInst;
 class VAArgInst;
 class Value;
 
diff --git a/llvm/include/llvm/Analysis/AssumeBundleQueries.h b/llvm/include/llvm/Analysis/AssumeBundleQueries.h
index df54fd2e9d655d9..f7a893708758c5a 100644
--- a/llvm/include/llvm/Analysis/AssumeBundleQueries.h
+++ b/llvm/include/llvm/Analysis/AssumeBundleQueries.h
@@ -21,7 +21,6 @@ namespace llvm {
 class AssumptionCache;
 class DominatorTree;
 class Instruction;
-class Value;
 
 /// Index of elements in the operand bundle.
 /// If the element exist it is guaranteed to be what is specified in this enum
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
index 179fd06addec875..5191a964403354d 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -25,7 +25,6 @@ namespace llvm {
 
 class BasicBlock;
 class BranchProbabilityInfo;
-class Function;
 class LoopInfo;
 class Module;
 class raw_ostream;
diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index eab7fec6b6e08f4..1ac5694b758feee 100644
--- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -22,7 +22,6 @@
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/BranchProbability.h"
-#include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <memory>
diff --git a/llvm/include/llvm/Analysis/CFGPrinter.h b/llvm/include/llvm/Analysis/CFGPrinter.h
index 7d4d3a574b9e81e..cd785331d1f1468 100644
--- a/llvm/include/llvm/Analysis/CFGPrinter.h
+++ b/llvm/include/llvm/Analysis/CFGPrinter.h
@@ -265,18 +265,19 @@ struct DOTGraphTraits<DOTFuncInfo *> : public DefaultDOTGraphTraits {
   /// Display the raw branch weights from PGO.
   std::string getEdgeAttributes(const BasicBlock *Node, const_succ_iterator I,
                                 DOTFuncInfo *CFGInfo) {
+    // If BPI is not provided do not display any edge attributes
+    if (!CFGInfo->showEdgeWeights())
+      return "";
+
     unsigned OpNo = I.getSuccessorIndex();
     const Instruction *TI = Node->getTerminator();
     BasicBlock *SuccBB = TI->getSuccessor(OpNo);
     auto BranchProb = CFGInfo->getBPI()->getEdgeProbability(Node, SuccBB);
     double WeightPercent = ((double)BranchProb.getNumerator()) /
                            ((double)BranchProb.getDenominator());
-
     std::string TTAttr =
         formatv("tooltip=\"{0} -> {1}\\nProbability {2:P}\" ", getBBName(Node),
                 getBBName(SuccBB), WeightPercent);
-    if (!CFGInfo->showEdgeWeights())
-      return TTAttr;
 
     if (TI->getNumSuccessors() == 1)
       return TTAttr + "penwidth=2";
diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h
index 406d3492136fcc7..15b7f226fd8283a 100644
--- a/llvm/include/llvm/Analysis/CGSCCPassManager.h
+++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h
@@ -99,7 +99,6 @@
 namespace llvm {
 
 class Function;
-class Value;
 template <typename T, unsigned int N> class SmallPriorityWorklist;
 struct CGSCCUpdateResult;
 
diff --git a/llvm/include/llvm/Analysis/CallGraph.h b/llvm/include/llvm/Analysis/CallGraph.h
index 887743774175dc3..f5ce2322b76d95a 100644
--- a/llvm/include/llvm/Analysis/CallGraph.h
+++ b/llvm/include/llvm/Analysis/CallGraph.h
@@ -46,7 +46,6 @@
 #define LLVM_ANALYSIS_CALLGRAPH_H
 
 #include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
diff --git a/llvm/include/llvm/Analysis/CaptureTracking.h b/llvm/include/llvm/Analysis/CaptureTracking.h
index 2825399e2cc4ddb..06a00d9ae789908 100644
--- a/llvm/include/llvm/Analysis/CaptureTracking.h
+++ b/llvm/include/llvm/Analysis/CaptureTracking.h
@@ -14,7 +14,6 @@
 #define LLVM_ANALYSIS_CAPTURETRACKING_H
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLFunctionalExtras.h"
 
 namespace llvm {
 
@@ -25,6 +24,7 @@ namespace llvm {
   class DominatorTree;
   class LoopInfo;
   class Function;
+  template <typename Fn> class function_ref;
 
   /// getDefaultMaxUsesToExploreForCaptureTracking - Return default value of
   /// the maximal number of uses to explore before giving up. It is used by
diff --git a/llvm/include/llvm/Analysis/CodeMetrics.h b/llvm/include/llvm/Analysis/CodeMetrics.h
index d09018daf954884..a51d923eb44ed98 100644
--- a/llvm/include/llvm/Analysis/CodeMetrics.h
+++ b/llvm/include/llvm/Analysis/CodeMetrics.h
@@ -20,7 +20,6 @@
 namespace llvm {
 class AssumptionCache;
 class BasicBlock;
-class Instruction;
 class Loop;
 class Function;
 template <class T> class SmallPtrSetImpl;
diff --git a/llvm/include/llvm/Analysis/ConstraintSystem.h b/llvm/include/llvm/Analysis/ConstraintSystem.h
index 7b02b618f7cb447..449852343964cad 100644
--- a/llvm/include/llvm/Analysis/ConstraintSystem.h
+++ b/llvm/include/llvm/Analysis/ConstraintSystem.h
@@ -9,7 +9,6 @@
 #ifndef LLVM_ANALYSIS_CONSTRAINTSYSTEM_H
 #define LLVM_ANALYSIS_CONSTRAINTSYSTEM_H
 
-#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
index f0e2aeb0f92f742..8707e83574dd84c 100644
--- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h
+++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
@@ -9,8 +9,6 @@
 #ifndef LLVM_ANALYSIS_CTXPROFANALYSIS_H
 #define LLVM_ANALYSIS_CTXPROFANALYSIS_H
 
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/ProfileData/PGOCtxProfReader.h"
 
@@ -79,7 +77,7 @@ class CtxProfAnalysis : public AnalysisInfoMixin<CtxProfAnalysis> {
 
 public:
   static AnalysisKey Key;
-  explicit CtxProfAnalysis(StringRef Profile) : Profile(Profile) {};
+  explicit CtxProfAnalysis(StringRef Profile = "");
 
   using Result = PGOContextualProfile;
 
diff --git a/llvm/include/llvm/Analysis/CycleAnalysis.h b/llvm/include/llvm/Analysis/CycleAnalysis.h
index ce939eff8ff8f32..6b45cece199913f 100644
--- a/llvm/include/llvm/Analysis/CycleAnalysis.h
+++ b/llvm/include/llvm/Analysis/CycleAnalysis.h
@@ -17,7 +17,6 @@
 
 #include "llvm/IR/CycleInfo.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/IR/SSAContext.h"
 #include "llvm/Pass.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h b/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h
index e5c6ae5db559eef..f325828495458b8 100644
--- a/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h
+++ b/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h
@@ -10,7 +10,6 @@
 #define LLVM_ANALYSIS_DXILMETADATA_H
 
 #include "llvm/IR/PassManager.h"
-#include "llvm/IR/Value.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/llvm/include/llvm/Analysis/DXILResource.h b/llvm/include/llvm/Analysis/DXILResource.h
index 3ba0ae5de61d515..14cf03c9a3acee2 100644
--- a/llvm/include/llvm/Analysis/DXILResource.h
+++ b/llvm/include/llvm/Analysis/DXILResource.h
@@ -10,15 +10,17 @@
 #define LLVM_ANALYSIS_DXILRESOURCE_H
 
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/IR/Value.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/DXILABI.h"
 
 namespace llvm {
 class CallInst;
+class LLVMContext;
 class MDTuple;
-class TargetExtType;
+class Value;
 
 namespace dxil {
 
diff --git a/llvm/include/llvm/Analysis/DomConditionCache.h b/llvm/include/llvm/Analysis/DomConditionCache.h
index 68e34f773af8ae8..ac25803143f49e8 100644
--- a/llvm/include/llvm/Analysis/DomConditionCache.h
+++ b/llvm/include/llvm/Analysis/DomConditionCache.h
@@ -19,9 +19,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/IR/ValueHandle.h"
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/Analysis/DomPrinter.h b/llvm/include/llvm/Analysis/DomPrinter.h
index 83fe721346ab2ed..80fdfcd56c366f6 100644
--- a/llvm/include/llvm/Analysis/DomPrinter.h
+++ b/llvm/include/llvm/Analysis/DomPrinter.h
@@ -17,7 +17,6 @@
 #include "llvm/Analysis/DOTGraphTraitsPass.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/IR/Dominators.h"
-#include "llvm/IR/PassManager.h"
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/Analysis/DomTreeUpdater.h b/llvm/include/llvm/Analysis/DomTreeUpdater.h
index e07d2bf2b0df17c..c120a6cc6ce5abe 100644
--- a/llvm/include/llvm/Analysis/DomTreeUpdater.h
+++ b/llvm/include/llvm/Analysis/DomTreeUpdater.h
@@ -14,12 +14,10 @@
 #ifndef LLVM_ANALYSIS_DOMTREEUPDATER_H
 #define LLVM_ANALYSIS_DOMTREEUPDATER_H
 
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Analysis/GenericDomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/Compiler.h"
-#include <cstddef>
 #include <functional>
 #include <vector>
 
diff --git a/llvm/include/llvm/Analysis/DominanceFrontier.h b/llvm/include/llvm/Analysis/DominanceFrontier.h
index 1339b4869bc29b1..8b3ab57ed4d07b5 100644
--- a/llvm/include/llvm/Analysis/DominanceFrontier.h
+++ b/llvm/include/llvm/Analysis/DominanceFrontier.h
@@ -20,7 +20,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/GenericDomTree.h"
diff --git a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
index f5fbbdcb7143d76..ee447d3e4ebb6ab 100644
--- a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
+++ b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
@@ -14,11 +14,12 @@
 #ifndef LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H
 #define LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H
 
-#include "llvm/ADT/iterator_range.h"
-#include "llvm/IR/InstrTypes.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
+class BasicBlock;
+class CallBase;
 class DominatorTree;
 class Function;
 class LoopInfo;
diff --git a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
index 0d19de6edc2a77a..9da465ddf75f9b8 100644
--- a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
+++ b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
@@ -57,7 +57,6 @@
 #include <optional>
 
 namespace llvm {
-class Module;
 
 namespace IRSimilarity {
 
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 5c7b613ac48c400..379b114b79cdfaf 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -23,7 +23,6 @@ namespace llvm {
 class AssumptionCache;
 class DemandedBits;
 class DominatorTree;
-class Instruction;
 class Loop;
 class PredicatedScalarEvolution;
 class ScalarEvolution;
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 5f36ee6f68abbb7..871a6e97861e297 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -12,7 +12,6 @@
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/LazyCallGraph.h"
-#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/PassManager.h"
 #include <memory>
 
diff --git a/llvm/include/llvm/Analysis/InlineOrder.h b/llvm/include/llvm/Analysis/InlineOrder.h
index a1d25a25b8568f7..2fa2d6091303ad8 100644
--- a/llvm/include/llvm/Analysis/InlineOrder.h
+++ b/llvm/include/llvm/Analysis/InlineOrder.h
@@ -9,12 +9,12 @@
 #ifndef LLVM_ANALYSIS_INLINEORDER_H
 #define LLVM_ANALYSIS_INLINEORDER_H
 
-#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/Analysis/InlineCost.h"
 #include <utility>
 
 namespace llvm {
 class CallBase;
+template <typename Fn> class function_ref;
 
 template <typename T> class InlineOrder {
 public:
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index 053367a78bfcf03..cf7d3e044188a60 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -39,7 +39,6 @@ namespace llvm {
 template <typename T, typename... TArgs> class AnalysisManager;
 template <class T> class ArrayRef;
 class AssumptionCache;
-class BinaryOperator;
 class CallBase;
 class DataLayout;
 class DominatorTree;
diff --git a/llvm/include/llvm/Analysis/InteractiveModelRunner.h b/llvm/include/llvm/Analysis/InteractiveModelRunner.h
index 680dc4249d80b17..798a249b9e4302b 100644
--- a/llvm/include/llvm/Analysis/InteractiveModelRunner.h
+++ b/llvm/include/llvm/Analysis/InteractiveModelRunner.h
@@ -13,9 +13,6 @@
 #include "llvm/Analysis/MLModelRunner.h"
 #include "llvm/Analysis/TensorSpec.h"
 #include "llvm/Analysis/Utils/TrainingLogger.h"
-#include "llvm/Config/llvm-config.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/raw_ostream.h"
 #include <system_error>
 
 namespace llvm {
diff --git a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
index a6d8b76b12aea24..8ad3c6bafc26d3c 100644
--- a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
@@ -21,7 +21,6 @@
 #include "llvm/Pass.h"
 
 namespace llvm {
-class AnalysisUsage;
 class Function;
 class LoopInfo;
 
@@ -125,5 +124,5 @@ class LazyBlockFrequencyInfoPass : public FunctionPass {
 
 /// Helper for client passes to initialize dependent passes for LBFI.
 void initializeLazyBFIPassPass(PassRegistry &Registry);
-}
+} // namespace llvm
 #endif
diff --git a/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h b/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h
index bad7423616b453b..fe1f7b25c0f159c 100644
--- a/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h
@@ -20,7 +20,6 @@
 #include "llvm/Pass.h"
 
 namespace llvm {
-class AnalysisUsage;
 class Function;
 class LoopInfo;
 class TargetLibraryInfo;
@@ -119,5 +118,5 @@ template <> struct BPIPassTrait<LazyBranchProbabilityInfoPass> {
     return P->getBPI();
   }
 };
-}
+} // namespace llvm
 #endif
diff --git a/llvm/include/llvm/Analysis/LazyCallGraph.h b/llvm/include/llvm/Analysis/LazyCallGraph.h
index a8bbf2c578af9b4..e67a0388bcaa605 100644
--- a/llvm/include/llvm/Analysis/LazyCallGraph.h
+++ b/llvm/include/llvm/Analysis/LazyCallGraph.h
@@ -55,11 +55,8 @@
 namespace llvm {
 
 class Constant;
-class Function;
 template <class GraphType> struct GraphTraits;
 class Module;
-class TargetLibraryInfo;
-class Value;
 
 /// A lazily constructed view of the call graph of a module.
 ///
@@ -111,7 +108,6 @@ class LazyCallGraph {
 public:
   class Node;
   class EdgeSequence;
-  class SCC;
   class RefSCC;
 
   /// A class used to represent edges in the call graph.
diff --git a/llvm/include/llvm/Analysis/LazyValueInfo.h b/llvm/include/llvm/Analysis/LazyValueInfo.h
index 3ca3e5a146e737f..52c7f68361a5b9f 100644
--- a/llvm/include/llvm/Analysis/LazyValueInfo.h
+++ b/llvm/include/llvm/Analysis/LazyValueInfo.h
@@ -22,11 +22,9 @@ namespace llvm {
   class AssumptionCache;
   class BasicBlock;
   class Constant;
-  class ConstantRange;
   class DataLayout;
   class DominatorTree;
   class Instruction;
-  class TargetLibraryInfo;
   class Value;
   class Use;
   class LazyValueInfoImpl;
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 1975d4aef9a6d70..87c70abba30fcdb 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -15,8 +15,7 @@
 #define LLVM_ANALYSIS_LOOPACCESSANALYSIS_H
 
 #include "llvm/ADT/EquivalenceClasses.h"
-#include "llvm/Analysis/LoopAnalysisManager.h"
-#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include <optional>
 #include <variant>
@@ -26,11 +25,8 @@ namespace llvm {
 class AAResults;
 class DataLayout;
 class Loop;
-class LoopAccessInfo;
 class raw_ostream;
-class SCEV;
-class SCEVUnionPredicate;
-class Value;
+class TargetTransformInfo;
 
 /// Collection of parameters shared beetween the Loop Vectorizer and the
 /// Loop Access Analysis.
diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h
index 054206925483c59..748b7e11cbde929 100644
--- a/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/llvm/include/llvm/Analysis/LoopInfo.h
@@ -14,13 +14,10 @@
 #define LLVM_ANALYSIS_LOOPINFO_H
 
 #include "llvm/ADT/GraphTraits.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/IR/CFG.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/GenericLoopInfo.h"
-#include <algorithm>
 #include <optional>
 #include <utility>
 
@@ -28,10 +25,8 @@ namespace llvm {
 
 class DominatorTree;
 class InductionDescriptor;
-class Instruction;
 class LoopInfo;
 class Loop;
-class MDNode;
 class MemorySSAUpdater;
 class ScalarEvolution;
 class raw_ostream;
diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
index 7d726a246ca3cb7..580dd5e95d76043 100644
--- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
@@ -13,10 +13,8 @@
 #include "llvm/Analysis/InlineAdvisor.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/MLModelRunner.h"
-#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/IR/PassManager.h"
 
-#include <deque>
 #include <map>
 #include <memory>
 #include <optional>
@@ -25,6 +23,7 @@ namespace llvm {
 class DiagnosticInfoOptimizationBase;
 class Module;
 class MLInlineAdvice;
+class ProfileSummaryInfo;
 
 class MLInlineAdvisor : public InlineAdvisor {
 public:
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index bb282a1b73d320b..7b48844cc9e8e9b 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -18,7 +18,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Analysis/TargetFolder.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/ValueHandle.h"
diff --git a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
index d5b2eb6253db959..decb33e6af6bcb5 100644
--- a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -29,7 +29,6 @@
 namespace llvm {
 
 class AssumptionCache;
-class BatchAAResults;
 class DominatorTree;
 class PHITransAddr;
 
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index 7d896c44f46795d..ea29e21bd18f2fa 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -37,7 +37,6 @@ class AnyMemTransferInst;
 class AnyMemIntrinsic;
 class TargetLibraryInfo;
 class VAArgInst;
-class Value;
 
 // Represents the size of a MemoryLocation. Logically, it's an
 // std::optional<uint63_t> that also carries a bit to represent whether the
diff --git a/llvm/include/llvm/Analysis/MemoryProfileInfo.h b/llvm/include/llvm/Analysis/MemoryProfileInfo.h
index 75ddf5cacf74d5c..edbce706953d18d 100644
--- a/llvm/include/llvm/Analysis/MemoryProfileInfo.h
+++ b/llvm/include/llvm/Analysis/MemoryProfileInfo.h
@@ -13,10 +13,7 @@
 #ifndef LLVM_ANALYSIS_MEMORYPROFILEINFO_H
 #define LLVM_ANALYSIS_MEMORYPROFILEINFO_H
 
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include <map>
 
diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h
index ac828021dd2ae1c..43fea6ba27ec4f4 100644
--- a/llvm/include/llvm/Analysis/MemorySSA.h
+++ b/llvm/include/llvm/Analysis/MemorySSA.h
@@ -108,16 +108,12 @@
 namespace llvm {
 
 template <class GraphType> struct GraphTraits;
-class BasicBlock;
 class Function;
 class Loop;
-class Instruction;
 class LLVMContext;
 class MemoryAccess;
 class MemorySSAWalker;
 class Module;
-class Use;
-class Value;
 class raw_ostream;
 
 namespace MSSAHelpers {
diff --git a/llvm/include/llvm/Analysis/MustExecute.h b/llvm/include/llvm/Analysis/MustExecute.h
index 468d94e7cd68b91..8ac3c5eb653cd4f 100644
--- a/llvm/include/llvm/Analysis/MustExecute.h
+++ b/llvm/include/llvm/Analysis/MustExecute.h
@@ -37,7 +37,6 @@ template <typename T> using GetterTy = std::function<T *(const Function &F)>;
 
 class BasicBlock;
 class DominatorTree;
-class Instruction;
 class Loop;
 class LoopInfo;
 class PostDominatorTree;
diff --git a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h
index 980b40500d7cc3e..035fb2234b24579 100644
--- a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h
+++ b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h
@@ -11,9 +11,9 @@
 #define LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H
 
 #include "llvm/Analysis/MLModelRunner.h"
-#include "llvm/Analysis/TensorSpec.h"
-#include "llvm/Config/llvm-config.h"
 namespace llvm {
+class TensorSpec;
+
 /// A pseudo model runner. We use it to store feature values when collecting
 /// logs for the default policy, in 'development' mode, but never ask it to
 /// 'run'.
diff --git a/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h b/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h
index 1a154c648fe6115..8195fb93938b063 100644
--- a/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h
@@ -23,7 +23,6 @@
 #define LLVM_ANALYSIS_OBJCARCALIASANALYSIS_H
 
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Pass.h"
 
 namespace llvm {
 namespace objcarc {
diff --git a/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h b/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
index f14344e743759f6..7b14e55782adbcb 100644
--- a/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
+++ b/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
@@ -22,8 +22,6 @@
 #include <optional>
 
 namespace llvm {
-class Function;
-class Value;
 
 /// The optimization diagnostic interface.
 ///
@@ -174,5 +172,5 @@ class OptimizationRemarkEmitterAnalysis
   /// Run the analysis pass over a function and produce BFI.
   Result run(Function &F, FunctionAnalysisManager &AM);
 };
-}
+} // namespace llvm
 #endif // LLVM_ANALYSIS_OPTIMIZATIONREMARKEMITTER_H
diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
index 73be9e1d74a33ba..ceae3e8a0ddb959 100644
--- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -15,18 +15,17 @@
 #define LLVM_ANALYSIS_PROFILESUMMARYINFO_H
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ProfileSummary.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/BlockFrequency.h"
 #include <memory>
 #include <optional>
 
 namespace llvm {
-class BasicBlock;
-class CallBase;
+class BlockFrequencyInfo;
 class MachineFunction;
 
 /// Analysis providing profile information.
diff --git a/llvm/include/llvm/Analysis/PtrUseVisitor.h b/llvm/include/llvm/Analysis/PtrUseVisitor.h
index 86206b2d5e9f883..b6cc14d2077af03 100644
--- a/llvm/include/llvm/Analysis/PtrUseVisitor.h
+++ b/llvm/include/llvm/Analysis/PtrUseVisitor.h
@@ -34,7 +34,6 @@
 
 namespace llvm {
 class DataLayout;
-class Use;
 
 namespace detail {
 
diff --git a/llvm/include/llvm/Analysis/RegionInfo.h b/llvm/include/llvm/Analysis/RegionInfo.h
index fc8df36ec287db9..474bd3a9483e717 100644
--- a/llvm/include/llvm/Analysis/RegionInfo.h
+++ b/llvm/include/llvm/Analysis/RegionInfo.h
@@ -41,7 +41,6 @@
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/iterator_range.h"
-#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
@@ -56,7 +55,6 @@
 
 namespace llvm {
 
-class BasicBlock;
 class DominanceFrontier;
 class Loop;
 class LoopInfo;
diff --git a/llvm/include/llvm/Analysis/RegionPrinter.h b/llvm/include/llvm/Analysis/RegionPrinter.h
index 501a5406236e926..814d085baf0ad64 100644
--- a/llvm/include/llvm/Analysis/RegionPrinter.h
+++ b/llvm/include/llvm/Analysis/RegionPrinter.h
@@ -14,13 +14,13 @@
 #ifndef LLVM_ANALYSIS_REGIONPRINTER_H
 #define LLVM_ANALYSIS_REGIONPRINTER_H
 
-#include "llvm/Analysis/DOTGraphTraitsPass.h"
-#include "llvm/Analysis/RegionInfo.h"
+#include "llvm/Support/DOTGraphTraits.h"
 
 namespace llvm {
   class FunctionPass;
   class Function;
   class RegionInfo;
+  class RegionNode;
 
   FunctionPass *createRegionViewerPass();
   FunctionPass *createRegionOnlyViewerPass();
diff --git a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h
index 0c5b566f60a4704..6f6ce763746a16c 100644
--- a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h
@@ -14,7 +14,6 @@
 
 namespace llvm {
 class CallBase;
-class Function;
 class LLVMContext;
 class Module;
 
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index f413e77877f9038..5154e2f6659c121 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -30,7 +30,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/ConstantRange.h"
-#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
@@ -51,9 +50,7 @@ class Constant;
 class ConstantInt;
 class DataLayout;
 class DominatorTree;
-class Function;
 class GEPOperator;
-class Instruction;
 class LLVMContext;
 class Loop;
 class LoopInfo;
@@ -64,7 +61,6 @@ class SCEVUnknown;
 class StructType;
 class TargetLibraryInfo;
 class Type;
-class Value;
 enum SCEVTypes : unsigned short;
 
 extern bool VerifySCEV;
diff --git a/llvm/include/llvm/Analysis/SyntheticCountsUtils.h b/llvm/include/llvm/Analysis/SyntheticCountsUtils.h
index 5a6c7dd53c7e557..8b61ac5ed8ca22d 100644
--- a/llvm/include/llvm/Analysis/SyntheticCountsUtils.h
+++ b/llvm/include/llvm/Analysis/SyntheticCountsUtils.h
@@ -13,8 +13,8 @@
 #ifndef LLVM_ANALYSIS_SYNTHETICCOUNTSUTILS_H
 #define LLVM_ANALYSIS_SYNTHETICCOUNTSUTILS_H
 
+#include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
-#include "llvm/Analysis/CallGraph.h"
 #include "llvm/Support/ScaledNumber.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index e1cb1e5c557eae9..5914324b286c05d 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -517,6 +517,16 @@ TLI_DEFINE_ENUM_INTERNAL(Exit)
 TLI_DEFINE_STRING_INTERNAL("_Exit")
 TLI_DEFINE_SIG_INTERNAL(Void, Int)
 
+/// void std::terminate();
+TLI_DEFINE_ENUM_INTERNAL(terminate)
+TLI_DEFINE_STRING_INTERNAL("_ZSt9terminatev")
+TLI_DEFINE_SIG_INTERNAL(Void)
+
+/// void __cxa_throw(void *, void *, void (*)(void *));
+TLI_DEFINE_ENUM_INTERNAL(cxa_throw)
+TLI_DEFINE_STRING_INTERNAL("__cxa_throw")
+TLI_DEFINE_SIG_INTERNAL(Void, Ptr, Ptr, Ptr)
+
 /// void __cxa_guard_abort(guard_t *guard);
 /// guard_t is int64_t in Itanium ABI or int32_t on ARM eabi.
 TLI_DEFINE_ENUM_INTERNAL(cxa_guard_abort)
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
index d0506a89b88ef46..8d848696bab874c 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
@@ -20,9 +20,6 @@
 namespace llvm {
 
 template <typename T> class ArrayRef;
-class Function;
-class Module;
-class Triple;
 
 /// Provides info so a possible vectorization of a function can be
 /// computed. Function 'VectorFnName' is equivalent to 'ScalarFnName'
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 38e8b9da213974b..b2124c6106198ec 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -22,7 +22,6 @@
 #define LLVM_ANALYSIS_TARGETTRANSFORMINFO_H
 
 #include "llvm/ADT/APInt.h"
-#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/IR/FMF.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/PassManager.h"
@@ -45,7 +44,6 @@ class AssumptionCache;
 class BlockFrequencyInfo;
 class DominatorTree;
 class BranchInst;
-class CallBase;
 class Function;
 class GlobalValue;
 class InstCombiner;
@@ -60,12 +58,11 @@ class ProfileSummaryInfo;
 class RecurrenceDescriptor;
 class SCEV;
 class ScalarEvolution;
+class SmallBitVector;
 class StoreInst;
 class SwitchInst;
 class TargetLibraryInfo;
 class Type;
-class User;
-class Value;
 class VPIntrinsic;
 struct KnownBits;
 
@@ -742,11 +739,6 @@ class TargetTransformInfo {
   /// cost should return false, otherwise return true.
   bool isNumRegsMajorCostOfLSR() const;
 
-  /// Return true if LSR should attempts to replace a use of an otherwise dead
-  /// primary IV in the latch condition with another IV available in the loop.
-  /// When successful, makes the primary IV dead.
-  bool shouldFoldTerminatingConditionAfterLSR() const;
-
   /// Return true if LSR should drop a found solution if it's calculated to be
   /// less profitable than the baseline.
   bool shouldDropLSRSolutionIfLessProfitable() const;
@@ -1891,7 +1883,6 @@ class TargetTransformInfo::Concept {
   virtual bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
                              const TargetTransformInfo::LSRCost &C2) = 0;
   virtual bool isNumRegsMajorCostOfLSR() = 0;
-  virtual bool shouldFoldTerminatingConditionAfterLSR() const = 0;
   virtual bool shouldDropLSRSolutionIfLessProfitable() const = 0;
   virtual bool isProfitableLSRChainElement(Instruction *I) = 0;
   virtual bool canMacroFuseCmp() = 0;
@@ -2370,9 +2361,6 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   bool isNumRegsMajorCostOfLSR() override {
     return Impl.isNumRegsMajorCostOfLSR();
   }
-  bool shouldFoldTerminatingConditionAfterLSR() const override {
-    return Impl.shouldFoldTerminatingConditionAfterLSR();
-  }
   bool shouldDropLSRSolutionIfLessProfitable() const override {
     return Impl.shouldDropLSRSolutionIfLessProfitable();
   }
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index d208a710bb27fda..11b07ac0b7fc47c 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -244,8 +244,6 @@ class TargetTransformInfoImplBase {
 
   bool isNumRegsMajorCostOfLSR() const { return true; }
 
-  bool shouldFoldTerminatingConditionAfterLSR() const { return false; }
-
   bool shouldDropLSRSolutionIfLessProfitable() const { return false; }
 
   bool isProfitableLSRChainElement(Instruction *I) const { return false; }
diff --git a/llvm/include/llvm/Analysis/UniformityAnalysis.h b/llvm/include/llvm/Analysis/UniformityAnalysis.h
index c38d100d88b8a7c..a067a9b42893cd0 100644
--- a/llvm/include/llvm/Analysis/UniformityAnalysis.h
+++ b/llvm/include/llvm/Analysis/UniformityAnalysis.h
@@ -15,7 +15,9 @@
 #define LLVM_ANALYSIS_UNIFORMITYANALYSIS_H
 
 #include "llvm/ADT/GenericUniformityInfo.h"
-#include "llvm/Analysis/CycleAnalysis.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/SSAContext.h"
+#include "llvm/Pass.h"
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/Analysis/ValueLattice.h b/llvm/include/llvm/Analysis/ValueLattice.h
index fa56d838a385980..a704cce0c082b9f 100644
--- a/llvm/include/llvm/Analysis/ValueLattice.h
+++ b/llvm/include/llvm/Analysis/ValueLattice.h
@@ -9,9 +9,8 @@
 #ifndef LLVM_ANALYSIS_VALUELATTICE_H
 #define LLVM_ANALYSIS_VALUELATTICE_H
 
-#include "llvm/IR/Constants.h"
 #include "llvm/IR/ConstantRange.h"
-#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Constants.h"
 
 //===----------------------------------------------------------------------===//
 //                               ValueLatticeElement
@@ -19,8 +18,6 @@
 
 namespace llvm {
 
-class Constant;
-
 /// This class represents lattice values for constants.
 ///
 /// FIXME: This is basically just for bringup, this can be made a lot more rich
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 96fa16970584da3..00ead1181d76258 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_ANALYSIS_VALUETRACKING_H
 #define LLVM_ANALYSIS_VALUETRACKING_H
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Analysis/SimplifyQuery.h"
 #include "llvm/Analysis/WithCache.h"
 #include "llvm/IR/Constants.h"
@@ -30,12 +29,9 @@ namespace llvm {
 
 class Operator;
 class AddOperator;
-class AllocaInst;
-class APInt;
 class AssumptionCache;
 class DominatorTree;
 class GEPOperator;
-class LoadInst;
 class WithOverflowInst;
 struct KnownBits;
 class Loop;
@@ -43,7 +39,7 @@ class LoopInfo;
 class MDNode;
 class StringRef;
 class TargetLibraryInfo;
-class Value;
+template <typename T> class ArrayRef;
 
 constexpr unsigned MaxAnalysisRecursionDepth = 6;
 
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 35283637027db84..e2dd4976f39065c 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -120,9 +120,7 @@ class DemandedBits;
 template <typename InstTy> class InterleaveGroup;
 class IRBuilderBase;
 class Loop;
-class ScalarEvolution;
 class TargetTransformInfo;
-class Type;
 class Value;
 
 namespace Intrinsic {
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 6827b5d95abfe01..d61d0961d5987d0 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -699,11 +699,19 @@ enum {
 
 // SPARC Specific e_flags
 enum : unsigned {
+  // ELF extension mask.
+  // All values are available for EM_SPARC32PLUS & EM_SPARCV9 objects, except
+  // EF_SPARC_32PLUS which is a EM_SPARC32PLUS-only flag.
+  //
+  // Note that those features are not mutually exclusive (one can set more than
+  // one flag in this group).
   EF_SPARC_EXT_MASK = 0xffff00,
   EF_SPARC_32PLUS = 0x000100,
   EF_SPARC_SUN_US1 = 0x000200,
   EF_SPARC_HAL_R1 = 0x000400,
   EF_SPARC_SUN_US3 = 0x000800,
+
+  // Memory model selection mask for EM_SPARCV9 objects.
   EF_SPARCV9_MM = 0x3,
   EF_SPARCV9_TSO = 0x0,
   EF_SPARCV9_PSO = 0x1,
@@ -1802,8 +1810,11 @@ enum : unsigned {
   AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI = 6,
   AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINIADDRDISC = 7,
   AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT = 8,
+  AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOTOS = 9,
+  AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_TYPEINFOVPTRDISCR = 10,
+  AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR = 11,
   AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST =
-      AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT,
+      AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR,
 };
 
 // x86 processor feature bits.
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 77ddc10e8a0e76d..217e3f1324f9c97 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -394,11 +394,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return TargetTransformInfoImplBase::isNumRegsMajorCostOfLSR();
   }
 
-  bool shouldFoldTerminatingConditionAfterLSR() const {
-    return TargetTransformInfoImplBase::
-        shouldFoldTerminatingConditionAfterLSR();
-  }
-
   bool shouldDropLSRSolutionIfLessProfitable() const {
     return TargetTransformInfoImplBase::shouldDropLSRSolutionIfLessProfitable();
   }
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h
index f826601544932d1..fa6a7be6cf6c321 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h
@@ -15,13 +15,13 @@
 #ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_H
 #define LLVM_CODEGEN_GLOBALISEL_COMBINER_H
 
+#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 
 namespace llvm {
 class MachineRegisterInfo;
-struct CombinerInfo;
 class GISelCSEInfo;
 class TargetPassConfig;
 class MachineFunction;
@@ -33,8 +33,12 @@ class MachineIRBuilder;
 /// TODO: Is it worth making this module-wide?
 class Combiner : public GIMatchTableExecutor {
 private:
+  using WorkListTy = GISelWorkList<512>;
+
   class WorkListMaintainer;
-  GISelWorkList<512> WorkList;
+  template <CombinerInfo::ObserverLevel Lvl> class WorkListMaintainerImpl;
+
+  WorkListTy WorkList;
 
   // We have a little hack here where keep the owned pointers private, and only
   // expose a reference. This has two purposes:
@@ -48,6 +52,8 @@ class Combiner : public GIMatchTableExecutor {
 
   bool HasSetupMF = false;
 
+  static bool tryDCE(MachineInstr &MI, MachineRegisterInfo &MRI);
+
 public:
   /// If CSEInfo is not null, then the Combiner will use CSEInfo as the observer
   /// and also create a CSEMIRBuilder. Pass nullptr if CSE is not needed.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index c76a54ed26ed562..32effc536eb35df 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -390,12 +390,6 @@ class CombinerHelper {
   /// Transform zext(trunc(x)) to x.
   bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg);
 
-  /// Transform [asz]ext([asz]ext(x)) to [asz]ext x.
-  bool matchCombineExtOfExt(MachineInstr &MI,
-                            std::tuple<Register, unsigned> &MatchInfo);
-  void applyCombineExtOfExt(MachineInstr &MI,
-                            std::tuple<Register, unsigned> &MatchInfo);
-
   /// Transform trunc (shl x, K) to shl (trunc x), K
   ///    if K < VT.getScalarSizeInBits().
   ///
@@ -903,6 +897,9 @@ class CombinerHelper {
   // fold ((A-C1)+C2) -> (A+(C2-C1))
   bool matchFoldAMinusC1PlusC2(const MachineInstr &MI, BuildFnTy &MatchInfo);
 
+  bool matchExtOfExt(const MachineInstr &FirstMI, const MachineInstr &SecondMI,
+                     BuildFnTy &MatchInfo);
+
 private:
   /// Checks for legality of an indexed variant of \p LdSt.
   bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
index 2b0eb71f8808277..67f95c962c58246 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
@@ -53,6 +53,31 @@ struct CombinerInfo {
   /// The maximum number of times the Combiner will iterate over the
   /// MachineFunction. Setting this to 0 enables fixed-point iteration.
   unsigned MaxIterations = 0;
+
+  enum class ObserverLevel {
+    /// Only retry combining created/changed instructions.
+    /// This replicates the legacy default Observer behavior for use with
+    /// fixed-point iteration.
+    Basic,
+    /// Enables Observer-based detection of dead instructions. This can save
+    /// some compile-time if full disabling of fixed-point iteration is not
+    /// desired. If the input IR doesn't contain dead instructions, consider
+    /// disabling \p EnableFullDCE.
+    DCE,
+    /// Enables Observer-based DCE and additional heuristics that retry
+    /// combining defined and used instructions of modified instructions.
+    /// This provides a good balance between compile-time and completeness of
+    /// combining without needing fixed-point iteration.
+    SinglePass,
+  };
+
+  /// Select how the Combiner acts on MIR changes.
+  ObserverLevel ObserverLvl = ObserverLevel::Basic;
+
+  /// Whether dead code elimination is performed before each Combiner iteration.
+  /// If Observer-based DCE is enabled, this controls if a full DCE pass is
+  /// performed before the first Combiner iteration.
+  bool EnableFullDCE = true;
 };
 } // namespace llvm
 
diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index 88ddd43a2a8913c..92efff93f60f89b 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -47,6 +47,8 @@ class BasicMatchContext {
   bool match(SDValue N, unsigned Opcode) const {
     return N->getOpcode() == Opcode;
   }
+
+  unsigned getNumOperands(SDValue N) const { return N->getNumOperands(); }
 };
 
 template <typename Pattern, typename MatchContext>
@@ -390,7 +392,8 @@ template <unsigned OpIdx, typename... OpndPreds> struct Operands_match {
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     // Returns false if there are more operands than predicates;
-    return N->getNumOperands() == OpIdx;
+    // Ignores the last two operands if both the Context and the Node are VP
+    return Ctx.getNumOperands(N) == OpIdx;
   }
 };
 
@@ -424,8 +427,9 @@ template <bool ExcludeChain> struct EffectiveOperands {
   unsigned Size = 0;
   unsigned FirstIndex = 0;
 
-  explicit EffectiveOperands(SDValue N) {
-    const unsigned TotalNumOps = N->getNumOperands();
+  template <typename MatchContext>
+  explicit EffectiveOperands(SDValue N, const MatchContext &Ctx) {
+    const unsigned TotalNumOps = Ctx.getNumOperands(N);
     FirstIndex = TotalNumOps;
     for (unsigned I = 0; I < TotalNumOps; ++I) {
       // Count the number of non-chain and non-glue nodes (we ignore chain
@@ -444,7 +448,9 @@ template <> struct EffectiveOperands<false> {
   unsigned Size = 0;
   unsigned FirstIndex = 0;
 
-  explicit EffectiveOperands(SDValue N) : Size(N->getNumOperands()) {}
+  template <typename MatchContext>
+  explicit EffectiveOperands(SDValue N, const MatchContext &Ctx)
+      : Size(Ctx.getNumOperands(N)) {}
 };
 
 // === Ternary operations ===
@@ -463,7 +469,7 @@ struct TernaryOpc_match {
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     if (sd_context_match(N, Ctx, m_Opc(Opcode))) {
-      EffectiveOperands<ExcludeChain> EO(N);
+      EffectiveOperands<ExcludeChain> EO(N, Ctx);
       assert(EO.Size == 3);
       return ((Op0.match(Ctx, N->getOperand(EO.FirstIndex)) &&
                Op1.match(Ctx, N->getOperand(EO.FirstIndex + 1))) ||
@@ -508,19 +514,28 @@ struct BinaryOpc_match {
   unsigned Opcode;
   LHS_P LHS;
   RHS_P RHS;
-
-  BinaryOpc_match(unsigned Opc, const LHS_P &L, const RHS_P &R)
-      : Opcode(Opc), LHS(L), RHS(R) {}
+  std::optional<SDNodeFlags> Flags;
+  BinaryOpc_match(unsigned Opc, const LHS_P &L, const RHS_P &R,
+                  std::optional<SDNodeFlags> Flgs = std::nullopt)
+      : Opcode(Opc), LHS(L), RHS(R), Flags(Flgs) {}
 
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     if (sd_context_match(N, Ctx, m_Opc(Opcode))) {
-      EffectiveOperands<ExcludeChain> EO(N);
+      EffectiveOperands<ExcludeChain> EO(N, Ctx);
       assert(EO.Size == 2);
-      return (LHS.match(Ctx, N->getOperand(EO.FirstIndex)) &&
-              RHS.match(Ctx, N->getOperand(EO.FirstIndex + 1))) ||
-             (Commutable && LHS.match(Ctx, N->getOperand(EO.FirstIndex + 1)) &&
-              RHS.match(Ctx, N->getOperand(EO.FirstIndex)));
+      if (!((LHS.match(Ctx, N->getOperand(EO.FirstIndex)) &&
+             RHS.match(Ctx, N->getOperand(EO.FirstIndex + 1))) ||
+            (Commutable && LHS.match(Ctx, N->getOperand(EO.FirstIndex + 1)) &&
+             RHS.match(Ctx, N->getOperand(EO.FirstIndex)))))
+        return false;
+
+      if (!Flags.has_value())
+        return true;
+
+      SDNodeFlags TmpFlags = *Flags;
+      TmpFlags.intersectWith(N->getFlags());
+      return TmpFlags == *Flags;
     }
 
     return false;
@@ -575,6 +590,19 @@ inline BinaryOpc_match<LHS, RHS, true> m_Or(const LHS &L, const RHS &R) {
   return BinaryOpc_match<LHS, RHS, true>(ISD::OR, L, R);
 }
 
+template <typename LHS, typename RHS>
+inline BinaryOpc_match<LHS, RHS, true> m_DisjointOr(const LHS &L,
+                                                    const RHS &R) {
+  SDNodeFlags Flags;
+  Flags.setDisjoint(true);
+  return BinaryOpc_match<LHS, RHS, true>(ISD::OR, L, R, Flags);
+}
+
+template <typename LHS, typename RHS>
+inline auto m_AddLike(const LHS &L, const RHS &R) {
+  return m_AnyOf(m_Add(L, R), m_DisjointOr(L, R));
+}
+
 template <typename LHS, typename RHS>
 inline BinaryOpc_match<LHS, RHS, true> m_Xor(const LHS &L, const RHS &R) {
   return BinaryOpc_match<LHS, RHS, true>(ISD::XOR, L, R);
@@ -661,15 +689,24 @@ inline BinaryOpc_match<LHS, RHS> m_FRem(const LHS &L, const RHS &R) {
 template <typename Opnd_P, bool ExcludeChain = false> struct UnaryOpc_match {
   unsigned Opcode;
   Opnd_P Opnd;
-
-  UnaryOpc_match(unsigned Opc, const Opnd_P &Op) : Opcode(Opc), Opnd(Op) {}
+  std::optional<SDNodeFlags> Flags;
+  UnaryOpc_match(unsigned Opc, const Opnd_P &Op,
+                 std::optional<SDNodeFlags> Flgs = std::nullopt)
+      : Opcode(Opc), Opnd(Op), Flags(Flgs) {}
 
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     if (sd_context_match(N, Ctx, m_Opc(Opcode))) {
-      EffectiveOperands<ExcludeChain> EO(N);
+      EffectiveOperands<ExcludeChain> EO(N, Ctx);
       assert(EO.Size == 1);
-      return Opnd.match(Ctx, N->getOperand(EO.FirstIndex));
+      if (!Opnd.match(Ctx, N->getOperand(EO.FirstIndex)))
+        return false;
+      if (!Flags.has_value())
+        return true;
+
+      SDNodeFlags TmpFlags = *Flags;
+      TmpFlags.intersectWith(N->getFlags());
+      return TmpFlags == *Flags;
     }
 
     return false;
@@ -695,6 +732,13 @@ template <typename Opnd> inline UnaryOpc_match<Opnd> m_ZExt(const Opnd &Op) {
   return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op);
 }
 
+template <typename Opnd>
+inline UnaryOpc_match<Opnd> m_NNegZExt(const Opnd &Op) {
+  SDNodeFlags Flags;
+  Flags.setNonNeg(true);
+  return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op, Flags);
+}
+
 template <typename Opnd> inline auto m_SExt(const Opnd &Op) {
   return UnaryOpc_match<Opnd>(ISD::SIGN_EXTEND, Op);
 }
@@ -719,6 +763,10 @@ template <typename Opnd> inline auto m_SExtOrSelf(const Opnd &Op) {
   return m_AnyOf(m_SExt(Op), Op);
 }
 
+template <typename Opnd> inline auto m_SExtLike(const Opnd &Op) {
+  return m_AnyOf(m_SExt(Op), m_NNegZExt(Op));
+}
+
 /// Match a aext or identity
 /// Allows to peek through optional extensions
 template <typename Opnd>
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index c762f09d55f1617..1514d92b36b3c24 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -672,6 +672,9 @@ class SelectionDAG {
   SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
                       bool isTarget = false, bool isOpaque = false);
 
+  SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
+                            bool isTarget = false, bool isOpaque = false);
+
   SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false,
                              bool IsOpaque = false);
 
@@ -1160,8 +1163,13 @@ class SelectionDAG {
                   SDValue N2, SDValue N3, const SDNodeFlags Flags);
   SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                   SDValue N2, SDValue N3, SDValue N4);
+  SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
+                  SDValue N2, SDValue N3, SDValue N4, const SDNodeFlags Flags);
   SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                   SDValue N2, SDValue N3, SDValue N4, SDValue N5);
+  SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
+                  SDValue N2, SDValue N3, SDValue N4, SDValue N5,
+                  const SDNodeFlags Flags);
 
   // Specialize again based on number of operands for nodes with a VTList
   // rather than a single VT.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 2f36c2e86b1c3a1..88549d9c9a28580 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -452,6 +452,20 @@ struct SDNodeFlags {
   bool hasNoFPExcept() const { return NoFPExcept; }
   bool hasUnpredictable() const { return Unpredictable; }
 
+  bool operator==(const SDNodeFlags &Other) const {
+    return NoUnsignedWrap == Other.NoUnsignedWrap &&
+           NoSignedWrap == Other.NoSignedWrap && Exact == Other.Exact &&
+           Disjoint == Other.Disjoint && NonNeg == Other.NonNeg &&
+           NoNaNs == Other.NoNaNs && NoInfs == Other.NoInfs &&
+           NoSignedZeros == Other.NoSignedZeros &&
+           AllowReciprocal == Other.AllowReciprocal &&
+           AllowContract == Other.AllowContract &&
+           ApproximateFuncs == Other.ApproximateFuncs &&
+           AllowReassociation == Other.AllowReassociation &&
+           NoFPExcept == Other.NoFPExcept &&
+           Unpredictable == Other.Unpredictable;
+  }
+
   /// Clear any flags in this flag set that aren't also set in Flags. All
   /// flags will be cleared if Flags are undefined.
   void intersectWith(const SDNodeFlags Flags) {
diff --git a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
index 99478e9f39e2260..9282c4a771afb2c 100644
--- a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
+++ b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
@@ -137,18 +137,21 @@ struct CaseBlock {
   SDLoc DL;
   DebugLoc DbgLoc;
 
-  // Branch weights.
+  // Branch weights and predictability.
   BranchProbability TrueProb, FalseProb;
+  bool IsUnpredictable;
 
   // Constructor for SelectionDAG.
   CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs,
             const Value *cmpmiddle, MachineBasicBlock *truebb,
             MachineBasicBlock *falsebb, MachineBasicBlock *me, SDLoc dl,
             BranchProbability trueprob = BranchProbability::getUnknown(),
-            BranchProbability falseprob = BranchProbability::getUnknown())
+            BranchProbability falseprob = BranchProbability::getUnknown(),
+            bool isunpredictable = false)
       : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs),
         TrueBB(truebb), FalseBB(falsebb), ThisBB(me), DL(dl),
-        TrueProb(trueprob), FalseProb(falseprob) {}
+        TrueProb(trueprob), FalseProb(falseprob),
+        IsUnpredictable(isunpredictable) {}
 
   // Constructor for GISel.
   CaseBlock(CmpInst::Predicate pred, bool nocmp, const Value *cmplhs,
@@ -156,10 +159,12 @@ struct CaseBlock {
             MachineBasicBlock *truebb, MachineBasicBlock *falsebb,
             MachineBasicBlock *me, DebugLoc dl,
             BranchProbability trueprob = BranchProbability::getUnknown(),
-            BranchProbability falseprob = BranchProbability::getUnknown())
+            BranchProbability falseprob = BranchProbability::getUnknown(),
+            bool isunpredictable = false)
       : PredInfo({pred, nocmp}), CmpLHS(cmplhs), CmpMHS(cmpmiddle),
         CmpRHS(cmprhs), TrueBB(truebb), FalseBB(falsebb), ThisBB(me),
-        DbgLoc(dl), TrueProb(trueprob), FalseProb(falseprob) {}
+        DbgLoc(dl), TrueProb(trueprob), FalseProb(falseprob),
+        IsUnpredictable(isunpredictable) {}
 };
 
 struct JumpTable {
diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h
index d00e0bed91a4579..2f5951e3ec3bce7 100644
--- a/llvm/include/llvm/CodeGen/TargetPassConfig.h
+++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h
@@ -140,6 +140,9 @@ class TargetPassConfig : public ImmutablePass {
   /// callers.
   bool RequireCodeGenSCCOrder = false;
 
+  /// Enable LoopTermFold immediately after LSR
+  bool EnableLoopTermFold = false;
+
   /// Add the actual instruction selection passes. This does not include
   /// preparation passes on IR.
   bool addCoreISelPasses();
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index b3976d7fbf41ff3..fb2060fa07f7243 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -30,7 +30,7 @@
 #include <type_traits>
 #include <utility>
 
-#ifdef _LIBCXXABI_COMPILER_CLANG
+#if defined(__clang__)
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wunused-template"
 #endif
@@ -199,8 +199,7 @@ class Node {
 
   Prec Precedence : 6;
 
-  // FIXME: Make these protected.
-public:
+protected:
   /// Tracks if this node has a component on its right side, in which case we
   /// need to call printRight.
   Cache RHSComponentCache : 2;
@@ -254,6 +253,9 @@ class Node {
   Kind getKind() const { return K; }
 
   Prec getPrecedence() const { return Precedence; }
+  Cache getRHSComponentCache() const { return RHSComponentCache; }
+  Cache getArrayCache() const { return ArrayCache; }
+  Cache getFunctionCache() const { return FunctionCache; }
 
   virtual bool hasRHSComponentSlow(OutputBuffer &) const { return false; }
   virtual bool hasArraySlow(OutputBuffer &) const { return false; }
@@ -423,8 +425,8 @@ class QualType final : public Node {
 
 public:
   QualType(const Node *Child_, Qualifiers Quals_)
-      : Node(KQualType, Child_->RHSComponentCache,
-             Child_->ArrayCache, Child_->FunctionCache),
+      : Node(KQualType, Child_->getRHSComponentCache(), Child_->getArrayCache(),
+             Child_->getFunctionCache()),
         Quals(Quals_), Child(Child_) {}
 
   Qualifiers getQuals() const { return Quals; }
@@ -553,8 +555,8 @@ struct AbiTagAttr : Node {
   std::string_view Tag;
 
   AbiTagAttr(Node *Base_, std::string_view Tag_)
-      : Node(KAbiTagAttr, Base_->RHSComponentCache, Base_->ArrayCache,
-             Base_->FunctionCache),
+      : Node(KAbiTagAttr, Base_->getRHSComponentCache(), Base_->getArrayCache(),
+             Base_->getFunctionCache()),
         Base(Base_), Tag(Tag_) {}
 
   template<typename Fn> void match(Fn F) const { F(Base, Tag); }
@@ -614,7 +616,7 @@ class PointerType final : public Node {
 
 public:
   PointerType(const Node *Pointee_)
-      : Node(KPointerType, Pointee_->RHSComponentCache),
+      : Node(KPointerType, Pointee_->getRHSComponentCache()),
         Pointee(Pointee_) {}
 
   const Node *getPointee() const { return Pointee; }
@@ -698,7 +700,7 @@ class ReferenceType : public Node {
 
 public:
   ReferenceType(const Node *Pointee_, ReferenceKind RK_)
-      : Node(KReferenceType, Pointee_->RHSComponentCache),
+      : Node(KReferenceType, Pointee_->getRHSComponentCache()),
         Pointee(Pointee_), RK(RK_) {}
 
   template<typename Fn> void match(Fn F) const { F(Pointee, RK); }
@@ -741,7 +743,7 @@ class PointerToMemberType final : public Node {
 
 public:
   PointerToMemberType(const Node *ClassType_, const Node *MemberType_)
-      : Node(KPointerToMemberType, MemberType_->RHSComponentCache),
+      : Node(KPointerToMemberType, MemberType_->getRHSComponentCache()),
         ClassType(ClassType_), MemberType(MemberType_) {}
 
   template<typename Fn> void match(Fn F) const { F(ClassType, MemberType); }
@@ -1382,16 +1384,14 @@ class ParameterPack final : public Node {
 public:
   ParameterPack(NodeArray Data_) : Node(KParameterPack), Data(Data_) {
     ArrayCache = FunctionCache = RHSComponentCache = Cache::Unknown;
-    if (std::all_of(Data.begin(), Data.end(), [](Node* P) {
-          return P->ArrayCache == Cache::No;
-        }))
+    if (std::all_of(Data.begin(), Data.end(),
+                    [](Node *P) { return P->getArrayCache() == Cache::No; }))
       ArrayCache = Cache::No;
-    if (std::all_of(Data.begin(), Data.end(), [](Node* P) {
-          return P->FunctionCache == Cache::No;
-        }))
+    if (std::all_of(Data.begin(), Data.end(),
+                    [](Node *P) { return P->getFunctionCache() == Cache::No; }))
       FunctionCache = Cache::No;
-    if (std::all_of(Data.begin(), Data.end(), [](Node* P) {
-          return P->RHSComponentCache == Cache::No;
+    if (std::all_of(Data.begin(), Data.end(), [](Node *P) {
+          return P->getRHSComponentCache() == Cache::No;
         }))
       RHSComponentCache = Cache::No;
   }
@@ -5947,7 +5947,7 @@ struct ManglingParser : AbstractManglingParser<ManglingParser<Alloc>, Alloc> {
 
 DEMANGLE_NAMESPACE_END
 
-#ifdef _LIBCXXABI_COMPILER_CLANG
+#if defined(__clang__)
 #pragma clang diagnostic pop
 #endif
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index f997faf1ebcb056..44db455f218a475 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -319,9 +319,6 @@ class StaticLibraryDefinitionGenerator : public DefinitionGenerator {
                                    Error &Err);
   Error buildObjectFilesMap();
 
-  static Expected<std::pair<size_t, size_t>>
-  getSliceRangeForArch(object::MachOUniversalBinary &UB, const Triple &TT);
-
   ObjectLayer &L;
   GetObjectFileInterface GetObjFileInterface;
   std::set<std::string> ImportedDynamicLibraries;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LoadRelocatableObject.h b/llvm/include/llvm/ExecutionEngine/Orc/LoadRelocatableObject.h
new file mode 100644
index 000000000000000..a6a2f41dcff7d61
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LoadRelocatableObject.h
@@ -0,0 +1,36 @@
+//===---- LoadRelocatableObject.h - Load relocatable objects ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A wrapper for `MemoryBuffer::getFile` / `MemoryBuffer::getFileSlice` that:
+//
+//   1. Adds file paths to errors by default.
+//   2. Checks architecture compatibility up-front.
+//   3. Handles MachO universal binaries, returning the MemoryBuffer for the
+//      requested slice only.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_LOADRELOCATABLEOBJECT_H
+#define LLVM_EXECUTIONENGINE_ORC_LOADRELOCATABLEOBJECT_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+namespace orc {
+
+// Load an object file compatible with the given triple (if given) from the
+// given path. May return a file slice if the path contains a universal binary.
+Expected<std::unique_ptr<MemoryBuffer>> loadRelocatableObject(StringRef Path,
+                                                              const Triple &TT);
+
+} // End namespace orc
+} // End namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_LOADRELOCATABLEOBJECT_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachO.h b/llvm/include/llvm/ExecutionEngine/Orc/MachO.h
new file mode 100644
index 000000000000000..58c04a9121c87b9
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MachO.h
@@ -0,0 +1,63 @@
+//===------------- MachO.h - MachO format utilities -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Contains utilities for load MachO relocatable object files.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_MACHO_H
+#define LLVM_EXECUTIONENGINE_ORC_MACHO_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+
+namespace object {
+
+class MachOUniversalBinary;
+
+} // namespace object
+
+namespace orc {
+
+/// Check that the given buffer contains a MachO object file compatible with the
+/// given triple.
+/// ObjIsSlice should be set to true if Obj is a slice of a universal binary
+/// (that fact will then be reported in the error messages).
+Expected<std::unique_ptr<MemoryBuffer>>
+checkMachORelocatableObject(std::unique_ptr<MemoryBuffer> Obj, const Triple &TT,
+                            bool ObjIsSlice);
+
+/// Load a relocatable object compatible with TT from Path.
+/// If Path is a universal binary, this function will return a buffer for the
+/// slice compatible with Triple (if one is present).
+Expected<std::unique_ptr<MemoryBuffer>>
+loadMachORelocatableObject(StringRef Path, const Triple &TT);
+
+/// Load a compatible relocatable object (if available) from a MachO universal
+/// binary.
+Expected<std::unique_ptr<MemoryBuffer>>
+loadMachORelocatableObjectFromUniversalBinary(
+    StringRef UBPath, std::unique_ptr<MemoryBuffer> UBBuf, const Triple &TT);
+
+/// Utility for identifying the file-slice compatible with TT in a universal
+/// binary.
+Expected<std::pair<size_t, size_t>>
+getMachOSliceRangeForTriple(object::MachOUniversalBinary &UB, const Triple &TT);
+
+/// Utility for identifying the file-slice compatible with TT in a universal
+/// binary.
+Expected<std::pair<size_t, size_t>>
+getMachOSliceRangeForTriple(MemoryBufferRef UBBuf, const Triple &TT);
+
+} // namespace orc
+} // namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_MACHO_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 9e4e7ebf2a5703e..4be0159fb1dd9f2 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -2195,7 +2195,7 @@ class OpenMPIRBuilder {
     /// The number of teams.
     ArrayRef<Value *> NumTeams;
     /// The number of threads.
-    Value *NumThreads = nullptr;
+    ArrayRef<Value *> NumThreads;
     /// The size of the dynamic shared memory.
     Value *DynCGGroupMem = nullptr;
     /// True if the kernel has 'no wait' clause.
@@ -2205,7 +2205,8 @@ class OpenMPIRBuilder {
     TargetKernelArgs() {}
     TargetKernelArgs(unsigned NumTargetItems, TargetDataRTArgs RTArgs,
                      Value *NumIterations, ArrayRef<Value *> NumTeams,
-                     Value *NumThreads, Value *DynCGGroupMem, bool HasNoWait)
+                     ArrayRef<Value *> NumThreads, Value *DynCGGroupMem,
+                     bool HasNoWait)
         : NumTargetItems(NumTargetItems), RTArgs(RTArgs),
           NumIterations(NumIterations), NumTeams(NumTeams),
           NumThreads(NumThreads), DynCGGroupMem(DynCGGroupMem),
@@ -2852,17 +2853,16 @@ class OpenMPIRBuilder {
   /// instructions for passed in target arguments where neccessary
   /// \param Dependencies A vector of DependData objects that carry
   // dependency information as passed in the depend clause
-  InsertPointTy createTarget(const LocationDescription &Loc,
-                             bool IsOffloadEntry,
-                             OpenMPIRBuilder::InsertPointTy AllocaIP,
-                             OpenMPIRBuilder::InsertPointTy CodeGenIP,
-                             TargetRegionEntryInfo &EntryInfo,
-                             ArrayRef<int32_t> NumTeams, int32_t NumThreads,
-                             SmallVectorImpl<Value *> &Inputs,
-                             GenMapInfoCallbackTy GenMapInfoCB,
-                             TargetBodyGenCallbackTy BodyGenCB,
-                             TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
-                             SmallVector<DependData> Dependencies = {});
+  InsertPointTy
+  createTarget(const LocationDescription &Loc, bool IsOffloadEntry,
+               OpenMPIRBuilder::InsertPointTy AllocaIP,
+               OpenMPIRBuilder::InsertPointTy CodeGenIP,
+               TargetRegionEntryInfo &EntryInfo, ArrayRef<int32_t> NumTeams,
+               ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Inputs,
+               GenMapInfoCallbackTy GenMapInfoCB,
+               TargetBodyGenCallbackTy BodyGenCB,
+               TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
+               SmallVector<DependData> Dependencies = {});
 
   /// Returns __kmpc_for_static_init_* runtime function for the specified
   /// size \a IVSize and sign \a IVSigned. Will create a distribute call
diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 1185939cd9c75b7..2f06bda6c30a51f 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -115,14 +115,6 @@ class DataLayout {
   // FIXME: `unsigned char` truncates the value parsed by `parseSpecifier`.
   SmallVector<unsigned char, 8> LegalIntWidths;
 
-  /// Type specifier used by some internal functions.
-  enum class TypeSpecifier {
-    Integer = 'i',
-    Float = 'f',
-    Vector = 'v',
-    Aggregate = 'a'
-  };
-
   /// Primitive type specifications. Sorted and uniqued by type bit width.
   SmallVector<PrimitiveSpec, 6> IntSpecs;
   SmallVector<PrimitiveSpec, 4> FloatSpecs;
@@ -145,19 +137,17 @@ class DataLayout {
   /// well-defined bitwise representation.
   SmallVector<unsigned, 8> NonIntegralAddressSpaces;
 
-  /// Attempts to set the specification for the given type.
-  /// Returns an error description on failure.
-  Error setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
-                         Align ABIAlign, Align PrefAlign);
+  /// Sets or updates the specification for the given primitive type.
+  void setPrimitiveSpec(char Specifier, uint32_t BitWidth, Align ABIAlign,
+                        Align PrefAlign);
 
   /// Searches for a pointer specification that matches the given address space.
   /// Returns the default address space specification if not found.
   const PointerSpec &getPointerSpec(uint32_t AddrSpace) const;
 
-  /// Attempts to set the specification for pointer in the given address space.
-  /// Returns an error description on failure.
-  Error setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth, Align ABIAlign,
-                       Align PrefAlign, uint32_t IndexBitWidth);
+  /// Sets or updates the specification for pointer in the given address space.
+  void setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth, Align ABIAlign,
+                      Align PrefAlign, uint32_t IndexBitWidth);
 
   /// Internal helper to get alignment for integer of given bitwidth.
   Align getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const;
@@ -165,9 +155,20 @@ class DataLayout {
   /// Internal helper method that returns requested alignment for type.
   Align getAlignment(Type *Ty, bool abi_or_pref) const;
 
-  /// Attempts to parse a target data specification string and reports an error
-  /// if the string is malformed.
-  Error parseSpecifier(StringRef Desc);
+  /// Attempts to parse primitive specification ('i', 'f', or 'v').
+  Error parsePrimitiveSpec(StringRef Spec);
+
+  /// Attempts to parse aggregate specification ('a').
+  Error parseAggregateSpec(StringRef Spec);
+
+  /// Attempts to parse pointer specification ('p').
+  Error parsePointerSpec(StringRef Spec);
+
+  /// Attempts to parse a single specification.
+  Error parseSpecification(StringRef Spec);
+
+  /// Attempts to parse a data layout string.
+  Error parseLayoutString(StringRef LayoutString);
 
 public:
   /// Constructs a DataLayout with default values.
@@ -188,7 +189,7 @@ class DataLayout {
 
   /// Parse a data layout string and return the layout. Return an error
   /// description on failure.
-  static Expected<DataLayout> parse(StringRef LayoutDescription);
+  static Expected<DataLayout> parse(StringRef LayoutString);
 
   /// Layout endianness...
   bool isLittleEndian() const { return !BigEndian; }
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 8c25467cc5e4b14..7c93c19a410e434 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -2343,6 +2343,8 @@ def int_amdgcn_buffer_wbinvl1_vol :
 // VI Intrinsics
 //===----------------------------------------------------------------------===//
 
+// The llvm.amdgcn.mov.dpp.i32 intrinsic represents the mov.dpp operation in AMDGPU.
+// This operation is being deprecated and can be replaced with llvm.amdgcn.update.dpp.i32.
 // llvm.amdgcn.mov.dpp.i32 <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl>
 def int_amdgcn_mov_dpp :
   Intrinsic<[llvm_anyint_ty],
@@ -2352,6 +2354,10 @@ def int_amdgcn_mov_dpp :
              ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>,
              ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, IntrNoCallback, IntrNoFree]>;
 
+// The llvm.amdgcn.update.dpp.i32 intrinsic represents the update.dpp operation in AMDGPU.
+// It takes an old value, a source operand, a DPP control operand, a row mask, a bank mask, and a bound control.
+// This operation is equivalent to a sequence of v_mov_b32 operations.
+// It is preferred over llvm.amdgcn.mov.dpp.i32 for future use.
 // llvm.amdgcn.update.dpp.i32 <old> <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl>
 // Should be equivalent to:
 // v_mov_b32 <dest> <old>
diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td
index c7ec0916f1d1f8f..d02eaa6d0dff691 100644
--- a/llvm/include/llvm/IR/IntrinsicsBPF.td
+++ b/llvm/include/llvm/IR/IntrinsicsBPF.td
@@ -13,11 +13,11 @@
 // Specialized loads from packet
 let TargetPrefix = "bpf" in {  // All intrinsics start with "llvm.bpf."
   def int_bpf_load_byte : ClangBuiltin<"__builtin_bpf_load_byte">,
-              Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
+              DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
   def int_bpf_load_half : ClangBuiltin<"__builtin_bpf_load_half">,
-              Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
+              DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
   def int_bpf_load_word : ClangBuiltin<"__builtin_bpf_load_word">,
-              Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
+              DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
   def int_bpf_pseudo : ClangBuiltin<"__builtin_bpf_pseudo">,
               Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty]>;
   def int_bpf_preserve_field_info : ClangBuiltin<"__builtin_bpf_preserve_field_info">,
diff --git a/llvm/include/llvm/IR/IntrinsicsLoongArch.td b/llvm/include/llvm/IR/IntrinsicsLoongArch.td
index 9002076e7aecea6..4621f1689b46e04 100644
--- a/llvm/include/llvm/IR/IntrinsicsLoongArch.td
+++ b/llvm/include/llvm/IR/IntrinsicsLoongArch.td
@@ -140,6 +140,11 @@ class VecInt<list<LLVMType> ret_types, list<LLVMType> param_types,
     : Intrinsic<ret_types, param_types, intr_properties>,
       ClangBuiltin<!subst("int_loongarch", "__builtin", NAME)>;
 
+class DefaultAttrsVecInt<list<LLVMType> ret_types, list<LLVMType> param_types,
+             list<IntrinsicProperty> intr_properties = []>
+    : DefaultAttrsIntrinsic<ret_types, param_types, intr_properties>,
+      ClangBuiltin<!subst("int_loongarch", "__builtin", NAME)>;
+
 //===----------------------------------------------------------------------===//
 // LSX
 
@@ -620,22 +625,22 @@ foreach inst = ["vfcmp_caf_d", "vfcmp_cun_d", "vfcmp_ceq_d", "vfcmp_cueq_d",
 
 // LSX load/store
 def int_loongarch_lsx_vld
-  : VecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lsx_vldx
-  : VecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i64_ty],
+  : DefaultAttrsVecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i64_ty],
            [IntrReadMem, IntrArgMemOnly]>;
 def int_loongarch_lsx_vldrepl_b
-  : VecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lsx_vldrepl_h
-  : VecInt<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lsx_vldrepl_w
-  : VecInt<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lsx_vldrepl_d
-  : VecInt<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 
 def int_loongarch_lsx_vst
@@ -1146,22 +1151,22 @@ def int_loongarch_lasx_xvpickve_d_f
 
 // LASX load/store
 def int_loongarch_lasx_xvld
-  : VecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lasx_xvldx
-  : VecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i64_ty],
+  : DefaultAttrsVecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i64_ty],
            [IntrReadMem, IntrArgMemOnly]>;
 def int_loongarch_lasx_xvldrepl_b
-  : VecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lasx_xvldrepl_h
-  : VecInt<[llvm_v16i16_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v16i16_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lasx_xvldrepl_w
-  : VecInt<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lasx_xvldrepl_d
-  : VecInt<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 
 def int_loongarch_lasx_xvst
diff --git a/llvm/include/llvm/IR/IntrinsicsMips.td b/llvm/include/llvm/IR/IntrinsicsMips.td
index 3056f37b9d87714..8d12786d52e126a 100644
--- a/llvm/include/llvm/IR/IntrinsicsMips.td
+++ b/llvm/include/llvm/IR/IntrinsicsMips.td
@@ -236,7 +236,7 @@ def int_mips_extpdp: ClangBuiltin<"__builtin_mips_extpdp">,
 def int_mips_wrdsp: ClangBuiltin<"__builtin_mips_wrdsp">,
   Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<1>>]>;
 def int_mips_rddsp: ClangBuiltin<"__builtin_mips_rddsp">,
-  Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrReadMem, ImmArg<ArgIndex<0>>]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrReadMem, ImmArg<ArgIndex<0>>]>;
 
 def int_mips_insv: ClangBuiltin<"__builtin_mips_insv">,
   Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrReadMem]>;
@@ -252,22 +252,22 @@ def int_mips_repl_ph: ClangBuiltin<"__builtin_mips_repl_ph">,
   Intrinsic<[mips_v2q15_ty], [llvm_i32_ty], [IntrNoMem]>;
 
 def int_mips_pick_qb: ClangBuiltin<"__builtin_mips_pick_qb">,
-  Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrReadMem]>;
+  DefaultAttrsIntrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrReadMem]>;
 def int_mips_pick_ph: ClangBuiltin<"__builtin_mips_pick_ph">,
-  Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrReadMem]>;
+  DefaultAttrsIntrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrReadMem]>;
 
 def int_mips_mthlip: ClangBuiltin<"__builtin_mips_mthlip">,
   Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty], []>;
 
 def int_mips_bposge32: ClangBuiltin<"__builtin_mips_bposge32">,
-  Intrinsic<[llvm_i32_ty], [], [IntrReadMem]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrReadMem]>;
 
 def int_mips_lbux: ClangBuiltin<"__builtin_mips_lbux">,
-  Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_lhx: ClangBuiltin<"__builtin_mips_lhx">,
-  Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_lwx: ClangBuiltin<"__builtin_mips_lwx">,
-  Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
 
 //===----------------------------------------------------------------------===//
 // MIPS DSP Rev 2
@@ -1259,23 +1259,23 @@ def int_mips_insve_d : ClangBuiltin<"__builtin_msa_insve_d">,
             [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_mips_ld_b : ClangBuiltin<"__builtin_msa_ld_b">,
-  Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_h : ClangBuiltin<"__builtin_msa_ld_h">,
-  Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_w : ClangBuiltin<"__builtin_msa_ld_w">,
-  Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_d : ClangBuiltin<"__builtin_msa_ld_d">,
-  Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 
 def int_mips_ldr_d : ClangBuiltin<"__builtin_msa_ldr_d">,
-  Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ldr_w : ClangBuiltin<"__builtin_msa_ldr_w">,
-  Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 
 def int_mips_ldi_b : ClangBuiltin<"__builtin_msa_ldi_b">,
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 65a3d2d0f943a77..8a333e7a559fe28 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1312,6 +1312,33 @@ let TargetPrefix = "nvvm" in {
   def int_nvvm_f2tf32_rna : ClangBuiltin<"__nvvm_f2tf32_rna">,
       Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
 
+  def int_nvvm_ff_to_e4m3x2_rn : ClangBuiltin<"__nvvm_ff_to_e4m3x2_rn">,
+      Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_ff_to_e4m3x2_rn_relu : ClangBuiltin<"__nvvm_ff_to_e4m3x2_rn_relu">,
+      Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_ff_to_e5m2x2_rn : ClangBuiltin<"__nvvm_ff_to_e5m2x2_rn">,
+      Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_ff_to_e5m2x2_rn_relu : ClangBuiltin<"__nvvm_ff_to_e5m2x2_rn_relu">,
+      Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+
+  def int_nvvm_f16x2_to_e4m3x2_rn : ClangBuiltin<"__nvvm_f16x2_to_e4m3x2_rn">,
+      Intrinsic<[llvm_i16_ty], [llvm_v2f16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f16x2_to_e4m3x2_rn_relu : ClangBuiltin<"__nvvm_f16x2_to_e4m3x2_rn_relu">,
+      Intrinsic<[llvm_i16_ty], [llvm_v2f16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f16x2_to_e5m2x2_rn : ClangBuiltin<"__nvvm_f16x2_to_e5m2x2_rn">,
+      Intrinsic<[llvm_i16_ty], [llvm_v2f16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f16x2_to_e5m2x2_rn_relu : ClangBuiltin<"__nvvm_f16x2_to_e5m2x2_rn_relu">,
+      Intrinsic<[llvm_i16_ty], [llvm_v2f16_ty], [IntrNoMem, IntrNoCallback]>;
+
+  def int_nvvm_e4m3x2_to_f16x2_rn : ClangBuiltin<"__nvvm_e4m3x2_to_f16x2_rn">,
+      Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_e4m3x2_to_f16x2_rn_relu : ClangBuiltin<"__nvvm_e4m3x2_to_f16x2_rn_relu">,
+      Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_e5m2x2_to_f16x2_rn : ClangBuiltin<"__nvvm_e5m2x2_to_f16x2_rn">,
+      Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_e5m2x2_to_f16x2_rn_relu : ClangBuiltin<"__nvvm_e5m2x2_to_f16x2_rn_relu">,
+      Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+
 //
 // Bitcast
 //
@@ -4729,7 +4756,7 @@ def int_nvvm_redux_sync_or : ClangBuiltin<"__nvvm_redux_sync_or">,
 class NVVM_WMMA_LD<WMMA_REGS Frag, string Layout, int WithStride>
   : Intrinsic<Frag.regs,
               !if(WithStride, [llvm_anyptr_ty, llvm_i32_ty], [llvm_anyptr_ty]),
-              [IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
+              [IntrWillReturn, IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
               WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.intr>;
 
 // WMMA.STORE.D
diff --git a/llvm/include/llvm/IR/IntrinsicsSystemZ.td b/llvm/include/llvm/IR/IntrinsicsSystemZ.td
index 9f4b905fedc7c17..4f925979cf85642 100644
--- a/llvm/include/llvm/IR/IntrinsicsSystemZ.td
+++ b/llvm/include/llvm/IR/IntrinsicsSystemZ.td
@@ -230,11 +230,11 @@ let TargetPrefix = "s390" in {
                                 [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
   def int_s390_vlbb : ClangBuiltin<"__builtin_s390_vlbb">,
-                      Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+                      DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
                                 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 
   def int_s390_vll : ClangBuiltin<"__builtin_s390_vll">,
-                     Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
+                     DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
                                [IntrReadMem, IntrArgMemOnly]>;
 
   def int_s390_vpdi : ClangBuiltin<"__builtin_s390_vpdi">,
@@ -399,7 +399,7 @@ let TargetPrefix = "s390" in {
 
   // Instructions from the Vector Packed Decimal Facility
   def int_s390_vlrl : ClangBuiltin<"__builtin_s390_vlrlr">,
-                      Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
+                      DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
                                 [IntrReadMem, IntrArgMemOnly]>;
 
   def int_s390_vstrl : ClangBuiltin<"__builtin_s390_vstrlr">,
diff --git a/llvm/include/llvm/IR/IntrinsicsVE.td b/llvm/include/llvm/IR/IntrinsicsVE.td
index 15b828b320ea2c4..633bc6603bf2311 100644
--- a/llvm/include/llvm/IR/IntrinsicsVE.td
+++ b/llvm/include/llvm/IR/IntrinsicsVE.td
@@ -3,10 +3,10 @@
 // VEL Intrinsic instructions.
 let TargetPrefix = "ve" in {
   def int_ve_vl_pack_f32p : ClangBuiltin<"__builtin_ve_vl_pack_f32p">,
-                            Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
+                            DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
                                       [IntrReadMem]>;
   def int_ve_vl_pack_f32a : ClangBuiltin<"__builtin_ve_vl_pack_f32a">,
-                            Intrinsic<[llvm_i64_ty], [llvm_ptr_ty],
+                            DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty],
                                       [IntrReadMem]>;
 
   def int_ve_vl_extract_vm512u :
diff --git a/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td b/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
index 554dd85572005e9..0c44dfb9076548e 100644
--- a/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
+++ b/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
@@ -1,35 +1,35 @@
-let TargetPrefix = "ve" in def int_ve_vl_vld_vssl : ClangBuiltin<"__builtin_ve_vl_vld_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld_vssvl : ClangBuiltin<"__builtin_ve_vl_vld_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu_vssl : ClangBuiltin<"__builtin_ve_vl_vldu_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld_vssl : ClangBuiltin<"__builtin_ve_vl_vld_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld_vssvl : ClangBuiltin<"__builtin_ve_vl_vld_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu_vssl : ClangBuiltin<"__builtin_ve_vl_vldu_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vst_vssl : ClangBuiltin<"__builtin_ve_vl_vst_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vst_vssml : ClangBuiltin<"__builtin_ve_vl_vst_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vstnc_vssl : ClangBuiltin<"__builtin_ve_vl_vstnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>;
@@ -1176,38 +1176,38 @@ let TargetPrefix = "ve" in def int_ve_vl_vror_vvl : ClangBuiltin<"__builtin_ve_v
 let TargetPrefix = "ve" in def int_ve_vl_vror_vvml : ClangBuiltin<"__builtin_ve_vl_vror_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvl : ClangBuiltin<"__builtin_ve_vl_vrxor_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvml : ClangBuiltin<"__builtin_ve_vl_vrxor_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssml : ClangBuiltin<"__builtin_ve_vl_vgt_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssml : ClangBuiltin<"__builtin_ve_vl_vgt_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssl : ClangBuiltin<"__builtin_ve_vl_vsc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssml : ClangBuiltin<"__builtin_ve_vl_vsc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vscnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vscnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index 4d2df1c44ebce5c..d03e532964f2878 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -346,7 +346,7 @@ def int_wasm_relaxed_dot_bf16x8_add_f32:
 // TODO: Replace these intrinsic with normal ISel patterns once the XXX
 // instructions are merged to the proposal.
 def int_wasm_loadf16_f32:
-  Intrinsic<[llvm_float_ty],
+  DefaultAttrsIntrinsic<[llvm_float_ty],
             [llvm_ptr_ty],
             [IntrReadMem, IntrArgMemOnly],
              "", [SDNPMemOperand]>;
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 0cd0d2edc236687..cc5e93c58f564ac 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -19,304 +19,305 @@ namespace llvm {
 class PassRegistry;
 
 /// Initialize all passes linked into the Core library.
-void initializeCore(PassRegistry&);
+void initializeCore(PassRegistry &);
 
 /// Initialize all passes linked into the TransformUtils library.
-void initializeTransformUtils(PassRegistry&);
+void initializeTransformUtils(PassRegistry &);
 
 /// Initialize all passes linked into the ScalarOpts library.
-void initializeScalarOpts(PassRegistry&);
+void initializeScalarOpts(PassRegistry &);
 
 /// Initialize all passes linked into the Vectorize library.
-void initializeVectorization(PassRegistry&);
+void initializeVectorization(PassRegistry &);
 
 /// Initialize all passes linked into the InstCombine library.
-void initializeInstCombine(PassRegistry&);
+void initializeInstCombine(PassRegistry &);
 
 /// Initialize all passes linked into the IPO library.
-void initializeIPO(PassRegistry&);
+void initializeIPO(PassRegistry &);
 
 /// Initialize all passes linked into the Analysis library.
-void initializeAnalysis(PassRegistry&);
+void initializeAnalysis(PassRegistry &);
 
 /// Initialize all passes linked into the CodeGen library.
-void initializeCodeGen(PassRegistry&);
+void initializeCodeGen(PassRegistry &);
 
 /// Initialize all passes linked into the GlobalISel library.
-void initializeGlobalISel(PassRegistry&);
+void initializeGlobalISel(PassRegistry &);
 
 /// Initialize all passes linked into the CodeGen library.
-void initializeTarget(PassRegistry&);
+void initializeTarget(PassRegistry &);
 
-void initializeAAResultsWrapperPassPass(PassRegistry&);
-void initializeAlwaysInlinerLegacyPassPass(PassRegistry&);
+void initializeAAResultsWrapperPassPass(PassRegistry &);
+void initializeAlwaysInlinerLegacyPassPass(PassRegistry &);
 void initializeAssignmentTrackingAnalysisPass(PassRegistry &);
-void initializeAssumptionCacheTrackerPass(PassRegistry&);
+void initializeAssumptionCacheTrackerPass(PassRegistry &);
 void initializeAtomicExpandLegacyPass(PassRegistry &);
 void initializeBasicBlockPathCloningPass(PassRegistry &);
 void initializeBasicBlockSectionsProfileReaderWrapperPassPass(PassRegistry &);
 void initializeBasicBlockSectionsPass(PassRegistry &);
-void initializeBarrierNoopPass(PassRegistry&);
-void initializeBasicAAWrapperPassPass(PassRegistry&);
-void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry&);
-void initializeBranchFolderPassPass(PassRegistry&);
-void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&);
-void initializeBranchRelaxationPass(PassRegistry&);
-void initializeBreakCriticalEdgesPass(PassRegistry&);
-void initializeBreakFalseDepsPass(PassRegistry&);
+void initializeBarrierNoopPass(PassRegistry &);
+void initializeBasicAAWrapperPassPass(PassRegistry &);
+void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry &);
+void initializeBranchFolderPassPass(PassRegistry &);
+void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &);
+void initializeBranchRelaxationPass(PassRegistry &);
+void initializeBreakCriticalEdgesPass(PassRegistry &);
+void initializeBreakFalseDepsPass(PassRegistry &);
 void initializeCanonicalizeFreezeInLoopsPass(PassRegistry &);
-void initializeCFGSimplifyPassPass(PassRegistry&);
-void initializeCFGuardPass(PassRegistry&);
-void initializeCFGuardLongjmpPass(PassRegistry&);
-void initializeCFIFixupPass(PassRegistry&);
-void initializeCFIInstrInserterPass(PassRegistry&);
+void initializeCFGSimplifyPassPass(PassRegistry &);
+void initializeCFGuardPass(PassRegistry &);
+void initializeCFGuardLongjmpPass(PassRegistry &);
+void initializeCFIFixupPass(PassRegistry &);
+void initializeCFIInstrInserterPass(PassRegistry &);
 void initializeCallBrPreparePass(PassRegistry &);
-void initializeCallGraphDOTPrinterPass(PassRegistry&);
-void initializeCallGraphViewerPass(PassRegistry&);
-void initializeCallGraphWrapperPassPass(PassRegistry&);
+void initializeCallGraphDOTPrinterPass(PassRegistry &);
+void initializeCallGraphViewerPass(PassRegistry &);
+void initializeCallGraphWrapperPassPass(PassRegistry &);
 void initializeCheckDebugMachineModulePass(PassRegistry &);
 void initializeCodeGenPrepareLegacyPassPass(PassRegistry &);
-void initializeComplexDeinterleavingLegacyPassPass(PassRegistry&);
-void initializeConstantHoistingLegacyPassPass(PassRegistry&);
+void initializeComplexDeinterleavingLegacyPassPass(PassRegistry &);
+void initializeConstantHoistingLegacyPassPass(PassRegistry &);
 void initializeCycleInfoWrapperPassPass(PassRegistry &);
-void initializeDAEPass(PassRegistry&);
-void initializeDAHPass(PassRegistry&);
-void initializeDCELegacyPassPass(PassRegistry&);
+void initializeDAEPass(PassRegistry &);
+void initializeDAHPass(PassRegistry &);
+void initializeDCELegacyPassPass(PassRegistry &);
 void initializeDXILMetadataAnalysisWrapperPassPass(PassRegistry &);
 void initializeDXILMetadataAnalysisWrapperPrinterPass(PassRegistry &);
 void initializeDXILResourceWrapperPassPass(PassRegistry &);
-void initializeDeadMachineInstructionElimPass(PassRegistry&);
+void initializeDeadMachineInstructionElimPass(PassRegistry &);
 void initializeDebugifyMachineModulePass(PassRegistry &);
-void initializeDependenceAnalysisWrapperPassPass(PassRegistry&);
-void initializeDetectDeadLanesPass(PassRegistry&);
+void initializeDependenceAnalysisWrapperPassPass(PassRegistry &);
+void initializeDetectDeadLanesPass(PassRegistry &);
 void initializeDomOnlyPrinterWrapperPassPass(PassRegistry &);
 void initializeDomOnlyViewerWrapperPassPass(PassRegistry &);
 void initializeDomPrinterWrapperPassPass(PassRegistry &);
 void initializeDomViewerWrapperPassPass(PassRegistry &);
-void initializeDominanceFrontierWrapperPassPass(PassRegistry&);
-void initializeDominatorTreeWrapperPassPass(PassRegistry&);
+void initializeDominanceFrontierWrapperPassPass(PassRegistry &);
+void initializeDominatorTreeWrapperPassPass(PassRegistry &);
 void initializeDwarfEHPrepareLegacyPassPass(PassRegistry &);
-void initializeEarlyCSELegacyPassPass(PassRegistry&);
-void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&);
-void initializeEarlyIfConverterPass(PassRegistry&);
+void initializeEarlyCSELegacyPassPass(PassRegistry &);
+void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry &);
+void initializeEarlyIfConverterPass(PassRegistry &);
 void initializeEarlyIfPredicatorPass(PassRegistry &);
-void initializeEarlyMachineLICMPass(PassRegistry&);
-void initializeEarlyTailDuplicatePass(PassRegistry&);
-void initializeEdgeBundlesPass(PassRegistry&);
+void initializeEarlyMachineLICMPass(PassRegistry &);
+void initializeEarlyTailDuplicatePass(PassRegistry &);
+void initializeEdgeBundlesPass(PassRegistry &);
 void initializeEHContGuardCatchretPass(PassRegistry &);
-void initializeExpandLargeFpConvertLegacyPassPass(PassRegistry&);
-void initializeExpandLargeDivRemLegacyPassPass(PassRegistry&);
+void initializeExpandLargeFpConvertLegacyPassPass(PassRegistry &);
+void initializeExpandLargeDivRemLegacyPassPass(PassRegistry &);
 void initializeExpandMemCmpLegacyPassPass(PassRegistry &);
-void initializeExpandPostRAPass(PassRegistry&);
-void initializeExpandReductionsPass(PassRegistry&);
+void initializeExpandPostRAPass(PassRegistry &);
+void initializeExpandReductionsPass(PassRegistry &);
 void initializeExpandVariadicsPass(PassRegistry &);
 void initializeExpandVectorPredicationPass(PassRegistry &);
-void initializeExternalAAWrapperPassPass(PassRegistry&);
-void initializeFEntryInserterPass(PassRegistry&);
-void initializeFinalizeISelPass(PassRegistry&);
-void initializeFinalizeMachineBundlesPass(PassRegistry&);
+void initializeExternalAAWrapperPassPass(PassRegistry &);
+void initializeFEntryInserterPass(PassRegistry &);
+void initializeFinalizeISelPass(PassRegistry &);
+void initializeFinalizeMachineBundlesPass(PassRegistry &);
 void initializeFixIrreduciblePass(PassRegistry &);
-void initializeFixupStatepointCallerSavedPass(PassRegistry&);
+void initializeFixupStatepointCallerSavedPass(PassRegistry &);
 void initializeFlattenCFGLegacyPassPass(PassRegistry &);
-void initializeFuncletLayoutPass(PassRegistry&);
+void initializeFuncletLayoutPass(PassRegistry &);
 void initializeGCEmptyBasicBlocksPass(PassRegistry &);
-void initializeGCMachineCodeAnalysisPass(PassRegistry&);
-void initializeGCModuleInfoPass(PassRegistry&);
-void initializeGVNLegacyPassPass(PassRegistry&);
-void initializeGlobalMergePass(PassRegistry&);
+void initializeGCMachineCodeAnalysisPass(PassRegistry &);
+void initializeGCModuleInfoPass(PassRegistry &);
+void initializeGVNLegacyPassPass(PassRegistry &);
+void initializeGlobalMergePass(PassRegistry &);
 void initializeGlobalsAAWrapperPassPass(PassRegistry &);
-void initializeHardwareLoopsLegacyPass(PassRegistry&);
+void initializeHardwareLoopsLegacyPass(PassRegistry &);
 void initializeMIRProfileLoaderPassPass(PassRegistry &);
-void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry&);
-void initializeIRTranslatorPass(PassRegistry&);
-void initializeIVUsersWrapperPassPass(PassRegistry&);
-void initializeIfConverterPass(PassRegistry&);
-void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry&);
-void initializeImplicitNullChecksPass(PassRegistry&);
+void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry &);
+void initializeIRTranslatorPass(PassRegistry &);
+void initializeIVUsersWrapperPassPass(PassRegistry &);
+void initializeIfConverterPass(PassRegistry &);
+void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry &);
+void initializeImplicitNullChecksPass(PassRegistry &);
 void initializeIndirectBrExpandLegacyPassPass(PassRegistry &);
-void initializeInferAddressSpacesPass(PassRegistry&);
+void initializeInferAddressSpacesPass(PassRegistry &);
 void initializeInstSimplifyLegacyPassPass(PassRegistry &);
-void initializeInstructionCombiningPassPass(PassRegistry&);
-void initializeInstructionSelectPass(PassRegistry&);
-void initializeInterleavedAccessPass(PassRegistry&);
+void initializeInstructionCombiningPassPass(PassRegistry &);
+void initializeInstructionSelectPass(PassRegistry &);
+void initializeInterleavedAccessPass(PassRegistry &);
 void initializeInterleavedLoadCombinePass(PassRegistry &);
-void initializeJMCInstrumenterPass(PassRegistry&);
+void initializeJMCInstrumenterPass(PassRegistry &);
 void initializeKCFIPass(PassRegistry &);
-void initializeLCSSAVerificationPassPass(PassRegistry&);
-void initializeLCSSAWrapperPassPass(PassRegistry&);
-void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&);
-void initializeLazyBranchProbabilityInfoPassPass(PassRegistry&);
-void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry&);
-void initializeLazyValueInfoWrapperPassPass(PassRegistry&);
-void initializeLegacyLICMPassPass(PassRegistry&);
-void initializeLegalizerPass(PassRegistry&);
+void initializeLCSSAVerificationPassPass(PassRegistry &);
+void initializeLCSSAWrapperPassPass(PassRegistry &);
+void initializeLazyBlockFrequencyInfoPassPass(PassRegistry &);
+void initializeLazyBranchProbabilityInfoPassPass(PassRegistry &);
+void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry &);
+void initializeLazyValueInfoWrapperPassPass(PassRegistry &);
+void initializeLegacyLICMPassPass(PassRegistry &);
+void initializeLegalizerPass(PassRegistry &);
 void initializeGISelCSEAnalysisWrapperPassPass(PassRegistry &);
 void initializeGISelKnownBitsAnalysisPass(PassRegistry &);
-void initializeLiveDebugValuesPass(PassRegistry&);
-void initializeLiveDebugVariablesPass(PassRegistry&);
+void initializeLiveDebugValuesPass(PassRegistry &);
+void initializeLiveDebugVariablesPass(PassRegistry &);
 void initializeLiveIntervalsWrapperPassPass(PassRegistry &);
-void initializeLiveRangeShrinkPass(PassRegistry&);
-void initializeLiveRegMatrixPass(PassRegistry&);
-void initializeLiveStacksPass(PassRegistry&);
+void initializeLiveRangeShrinkPass(PassRegistry &);
+void initializeLiveRegMatrixPass(PassRegistry &);
+void initializeLiveStacksPass(PassRegistry &);
 void initializeLiveVariablesWrapperPassPass(PassRegistry &);
 void initializeLoadStoreOptPass(PassRegistry &);
-void initializeLoadStoreVectorizerLegacyPassPass(PassRegistry&);
-void initializeLocalStackSlotPassPass(PassRegistry&);
-void initializeLocalizerPass(PassRegistry&);
-void initializeLoopDataPrefetchLegacyPassPass(PassRegistry&);
+void initializeLoadStoreVectorizerLegacyPassPass(PassRegistry &);
+void initializeLocalStackSlotPassPass(PassRegistry &);
+void initializeLocalizerPass(PassRegistry &);
+void initializeLoopDataPrefetchLegacyPassPass(PassRegistry &);
 void initializeLoopExtractorLegacyPassPass(PassRegistry &);
-void initializeLoopInfoWrapperPassPass(PassRegistry&);
-void initializeLoopPassPass(PassRegistry&);
-void initializeLoopSimplifyPass(PassRegistry&);
-void initializeLoopStrengthReducePass(PassRegistry&);
-void initializeLoopUnrollPass(PassRegistry&);
-void initializeLowerAtomicLegacyPassPass(PassRegistry&);
-void initializeLowerConstantIntrinsicsPass(PassRegistry&);
-void initializeLowerEmuTLSPass(PassRegistry&);
+void initializeLoopInfoWrapperPassPass(PassRegistry &);
+void initializeLoopPassPass(PassRegistry &);
+void initializeLoopSimplifyPass(PassRegistry &);
+void initializeLoopStrengthReducePass(PassRegistry &);
+void initializeLoopTermFoldPass(PassRegistry &);
+void initializeLoopUnrollPass(PassRegistry &);
+void initializeLowerAtomicLegacyPassPass(PassRegistry &);
+void initializeLowerConstantIntrinsicsPass(PassRegistry &);
+void initializeLowerEmuTLSPass(PassRegistry &);
 void initializeLowerGlobalDtorsLegacyPassPass(PassRegistry &);
-void initializeLowerIntrinsicsPass(PassRegistry&);
-void initializeLowerInvokeLegacyPassPass(PassRegistry&);
+void initializeLowerIntrinsicsPass(PassRegistry &);
+void initializeLowerInvokeLegacyPassPass(PassRegistry &);
 void initializeLowerSwitchLegacyPassPass(PassRegistry &);
 void initializeKCFIPass(PassRegistry &);
 void initializeMIRAddFSDiscriminatorsPass(PassRegistry &);
 void initializeMIRCanonicalizerPass(PassRegistry &);
 void initializeMIRNamerPass(PassRegistry &);
-void initializeMIRPrintingPassPass(PassRegistry&);
+void initializeMIRPrintingPassPass(PassRegistry &);
 void initializeMachineBlockFrequencyInfoWrapperPassPass(PassRegistry &);
-void initializeMachineBlockPlacementPass(PassRegistry&);
-void initializeMachineBlockPlacementStatsPass(PassRegistry&);
+void initializeMachineBlockPlacementPass(PassRegistry &);
+void initializeMachineBlockPlacementStatsPass(PassRegistry &);
 void initializeMachineBranchProbabilityInfoWrapperPassPass(PassRegistry &);
 void initializeMachineCFGPrinterPass(PassRegistry &);
-void initializeMachineCSEPass(PassRegistry&);
-void initializeMachineCombinerPass(PassRegistry&);
-void initializeMachineCopyPropagationPass(PassRegistry&);
+void initializeMachineCSEPass(PassRegistry &);
+void initializeMachineCombinerPass(PassRegistry &);
+void initializeMachineCopyPropagationPass(PassRegistry &);
 void initializeMachineCycleInfoPrinterPassPass(PassRegistry &);
 void initializeMachineCycleInfoWrapperPassPass(PassRegistry &);
-void initializeMachineDominanceFrontierPass(PassRegistry&);
+void initializeMachineDominanceFrontierPass(PassRegistry &);
 void initializeMachineDominatorTreeWrapperPassPass(PassRegistry &);
-void initializeMachineFunctionPrinterPassPass(PassRegistry&);
+void initializeMachineFunctionPrinterPassPass(PassRegistry &);
 void initializeMachineFunctionSplitterPass(PassRegistry &);
-void initializeMachineLateInstrsCleanupPass(PassRegistry&);
-void initializeMachineLICMPass(PassRegistry&);
+void initializeMachineLateInstrsCleanupPass(PassRegistry &);
+void initializeMachineLICMPass(PassRegistry &);
 void initializeMachineLoopInfoWrapperPassPass(PassRegistry &);
 void initializeMachineModuleInfoWrapperPassPass(PassRegistry &);
-void initializeMachineOptimizationRemarkEmitterPassPass(PassRegistry&);
-void initializeMachineOutlinerPass(PassRegistry&);
-void initializeMachinePipelinerPass(PassRegistry&);
+void initializeMachineOptimizationRemarkEmitterPassPass(PassRegistry &);
+void initializeMachineOutlinerPass(PassRegistry &);
+void initializeMachinePipelinerPass(PassRegistry &);
 void initializeMachinePostDominatorTreeWrapperPassPass(PassRegistry &);
-void initializeMachineRegionInfoPassPass(PassRegistry&);
+void initializeMachineRegionInfoPassPass(PassRegistry &);
 void initializeMachineSanitizerBinaryMetadataPass(PassRegistry &);
-void initializeMachineSchedulerPass(PassRegistry&);
-void initializeMachineSinkingPass(PassRegistry&);
-void initializeMachineTraceMetricsPass(PassRegistry&);
+void initializeMachineSchedulerPass(PassRegistry &);
+void initializeMachineSinkingPass(PassRegistry &);
+void initializeMachineTraceMetricsPass(PassRegistry &);
 void initializeMachineUniformityInfoPrinterPassPass(PassRegistry &);
 void initializeMachineUniformityAnalysisPassPass(PassRegistry &);
 void initializeMachineVerifierLegacyPassPass(PassRegistry &);
-void initializeMemoryDependenceWrapperPassPass(PassRegistry&);
-void initializeMemorySSAWrapperPassPass(PassRegistry&);
+void initializeMemoryDependenceWrapperPassPass(PassRegistry &);
+void initializeMemorySSAWrapperPassPass(PassRegistry &);
 void initializeMergeICmpsLegacyPassPass(PassRegistry &);
-void initializeModuleSummaryIndexWrapperPassPass(PassRegistry&);
-void initializeModuloScheduleTestPass(PassRegistry&);
-void initializeNaryReassociateLegacyPassPass(PassRegistry&);
+void initializeModuleSummaryIndexWrapperPassPass(PassRegistry &);
+void initializeModuloScheduleTestPass(PassRegistry &);
+void initializeNaryReassociateLegacyPassPass(PassRegistry &);
 void initializeObjCARCContractLegacyPassPass(PassRegistry &);
-void initializeOptimizationRemarkEmitterWrapperPassPass(PassRegistry&);
-void initializeOptimizePHIsPass(PassRegistry&);
-void initializePEIPass(PassRegistry&);
-void initializePHIEliminationPass(PassRegistry&);
-void initializePartiallyInlineLibCallsLegacyPassPass(PassRegistry&);
-void initializePatchableFunctionPass(PassRegistry&);
-void initializePeepholeOptimizerPass(PassRegistry&);
-void initializePhiValuesWrapperPassPass(PassRegistry&);
-void initializePhysicalRegisterUsageInfoPass(PassRegistry&);
+void initializeOptimizationRemarkEmitterWrapperPassPass(PassRegistry &);
+void initializeOptimizePHIsPass(PassRegistry &);
+void initializePEIPass(PassRegistry &);
+void initializePHIEliminationPass(PassRegistry &);
+void initializePartiallyInlineLibCallsLegacyPassPass(PassRegistry &);
+void initializePatchableFunctionPass(PassRegistry &);
+void initializePeepholeOptimizerPass(PassRegistry &);
+void initializePhiValuesWrapperPassPass(PassRegistry &);
+void initializePhysicalRegisterUsageInfoPass(PassRegistry &);
 void initializePlaceBackedgeSafepointsLegacyPassPass(PassRegistry &);
 void initializePostDomOnlyPrinterWrapperPassPass(PassRegistry &);
 void initializePostDomOnlyViewerWrapperPassPass(PassRegistry &);
 void initializePostDomPrinterWrapperPassPass(PassRegistry &);
 void initializePostDomViewerWrapperPassPass(PassRegistry &);
-void initializePostDominatorTreeWrapperPassPass(PassRegistry&);
-void initializePostInlineEntryExitInstrumenterPass(PassRegistry&);
-void initializePostMachineSchedulerPass(PassRegistry&);
-void initializePostRAHazardRecognizerPass(PassRegistry&);
-void initializePostRAMachineSinkingPass(PassRegistry&);
-void initializePostRASchedulerPass(PassRegistry&);
-void initializePreISelIntrinsicLoweringLegacyPassPass(PassRegistry&);
-void initializePrintFunctionPassWrapperPass(PassRegistry&);
-void initializePrintModulePassWrapperPass(PassRegistry&);
-void initializeProcessImplicitDefsPass(PassRegistry&);
-void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&);
-void initializePromoteLegacyPassPass(PassRegistry&);
-void initializeRABasicPass(PassRegistry&);
+void initializePostDominatorTreeWrapperPassPass(PassRegistry &);
+void initializePostInlineEntryExitInstrumenterPass(PassRegistry &);
+void initializePostMachineSchedulerPass(PassRegistry &);
+void initializePostRAHazardRecognizerPass(PassRegistry &);
+void initializePostRAMachineSinkingPass(PassRegistry &);
+void initializePostRASchedulerPass(PassRegistry &);
+void initializePreISelIntrinsicLoweringLegacyPassPass(PassRegistry &);
+void initializePrintFunctionPassWrapperPass(PassRegistry &);
+void initializePrintModulePassWrapperPass(PassRegistry &);
+void initializeProcessImplicitDefsPass(PassRegistry &);
+void initializeProfileSummaryInfoWrapperPassPass(PassRegistry &);
+void initializePromoteLegacyPassPass(PassRegistry &);
+void initializeRABasicPass(PassRegistry &);
 void initializePseudoProbeInserterPass(PassRegistry &);
-void initializeRAGreedyPass(PassRegistry&);
-void initializeReachingDefAnalysisPass(PassRegistry&);
-void initializeReassociateLegacyPassPass(PassRegistry&);
+void initializeRAGreedyPass(PassRegistry &);
+void initializeReachingDefAnalysisPass(PassRegistry &);
+void initializeReassociateLegacyPassPass(PassRegistry &);
 void initializeRegAllocEvictionAdvisorAnalysisPass(PassRegistry &);
-void initializeRegAllocFastPass(PassRegistry&);
+void initializeRegAllocFastPass(PassRegistry &);
 void initializeRegAllocPriorityAdvisorAnalysisPass(PassRegistry &);
 void initializeRegAllocScoringPass(PassRegistry &);
-void initializeRegBankSelectPass(PassRegistry&);
-void initializeRegUsageInfoCollectorPass(PassRegistry&);
-void initializeRegUsageInfoPropagationPass(PassRegistry&);
-void initializeRegionInfoPassPass(PassRegistry&);
-void initializeRegionOnlyPrinterPass(PassRegistry&);
-void initializeRegionOnlyViewerPass(PassRegistry&);
-void initializeRegionPrinterPass(PassRegistry&);
-void initializeRegionViewerPass(PassRegistry&);
-void initializeRegisterCoalescerPass(PassRegistry&);
-void initializeRemoveRedundantDebugValuesPass(PassRegistry&);
-void initializeRenameIndependentSubregsPass(PassRegistry&);
+void initializeRegBankSelectPass(PassRegistry &);
+void initializeRegUsageInfoCollectorPass(PassRegistry &);
+void initializeRegUsageInfoPropagationPass(PassRegistry &);
+void initializeRegionInfoPassPass(PassRegistry &);
+void initializeRegionOnlyPrinterPass(PassRegistry &);
+void initializeRegionOnlyViewerPass(PassRegistry &);
+void initializeRegionPrinterPass(PassRegistry &);
+void initializeRegionViewerPass(PassRegistry &);
+void initializeRegisterCoalescerPass(PassRegistry &);
+void initializeRemoveRedundantDebugValuesPass(PassRegistry &);
+void initializeRenameIndependentSubregsPass(PassRegistry &);
 void initializeReplaceWithVeclibLegacyPass(PassRegistry &);
-void initializeResetMachineFunctionPass(PassRegistry&);
-void initializeSCEVAAWrapperPassPass(PassRegistry&);
-void initializeSROALegacyPassPass(PassRegistry&);
-void initializeSafeStackLegacyPassPass(PassRegistry&);
-void initializeSafepointIRVerifierPass(PassRegistry&);
+void initializeResetMachineFunctionPass(PassRegistry &);
+void initializeSCEVAAWrapperPassPass(PassRegistry &);
+void initializeSROALegacyPassPass(PassRegistry &);
+void initializeSafeStackLegacyPassPass(PassRegistry &);
+void initializeSafepointIRVerifierPass(PassRegistry &);
 void initializeSelectOptimizePass(PassRegistry &);
-void initializeScalarEvolutionWrapperPassPass(PassRegistry&);
+void initializeScalarEvolutionWrapperPassPass(PassRegistry &);
 void initializeScalarizeMaskedMemIntrinLegacyPassPass(PassRegistry &);
-void initializeScavengerTestPass(PassRegistry&);
-void initializeScopedNoAliasAAWrapperPassPass(PassRegistry&);
+void initializeScavengerTestPass(PassRegistry &);
+void initializeScopedNoAliasAAWrapperPassPass(PassRegistry &);
 void initializeSeparateConstOffsetFromGEPLegacyPassPass(PassRegistry &);
-void initializeShadowStackGCLoweringPass(PassRegistry&);
-void initializeShrinkWrapPass(PassRegistry&);
-void initializeSingleLoopExtractorPass(PassRegistry&);
-void initializeSinkingLegacyPassPass(PassRegistry&);
-void initializeSjLjEHPreparePass(PassRegistry&);
+void initializeShadowStackGCLoweringPass(PassRegistry &);
+void initializeShrinkWrapPass(PassRegistry &);
+void initializeSingleLoopExtractorPass(PassRegistry &);
+void initializeSinkingLegacyPassPass(PassRegistry &);
+void initializeSjLjEHPreparePass(PassRegistry &);
 void initializeSlotIndexesWrapperPassPass(PassRegistry &);
-void initializeSpeculativeExecutionLegacyPassPass(PassRegistry&);
-void initializeSpillPlacementPass(PassRegistry&);
-void initializeStackColoringPass(PassRegistry&);
+void initializeSpeculativeExecutionLegacyPassPass(PassRegistry &);
+void initializeSpillPlacementPass(PassRegistry &);
+void initializeStackColoringPass(PassRegistry &);
 void initializeStackFrameLayoutAnalysisPassPass(PassRegistry &);
-void initializeStackMapLivenessPass(PassRegistry&);
-void initializeStackProtectorPass(PassRegistry&);
+void initializeStackMapLivenessPass(PassRegistry &);
+void initializeStackProtectorPass(PassRegistry &);
 void initializeStackSafetyGlobalInfoWrapperPassPass(PassRegistry &);
 void initializeStackSafetyInfoWrapperPassPass(PassRegistry &);
-void initializeStackSlotColoringPass(PassRegistry&);
+void initializeStackSlotColoringPass(PassRegistry &);
 void initializeStraightLineStrengthReduceLegacyPassPass(PassRegistry &);
 void initializeStripDebugMachineModulePass(PassRegistry &);
 void initializeStructurizeCFGLegacyPassPass(PassRegistry &);
-void initializeTailCallElimPass(PassRegistry&);
-void initializeTailDuplicatePass(PassRegistry&);
-void initializeTargetLibraryInfoWrapperPassPass(PassRegistry&);
-void initializeTargetPassConfigPass(PassRegistry&);
-void initializeTargetTransformInfoWrapperPassPass(PassRegistry&);
+void initializeTailCallElimPass(PassRegistry &);
+void initializeTailDuplicatePass(PassRegistry &);
+void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &);
+void initializeTargetPassConfigPass(PassRegistry &);
+void initializeTargetTransformInfoWrapperPassPass(PassRegistry &);
 void initializeTLSVariableHoistLegacyPassPass(PassRegistry &);
 void initializeTwoAddressInstructionLegacyPassPass(PassRegistry &);
-void initializeTypeBasedAAWrapperPassPass(PassRegistry&);
-void initializeTypePromotionLegacyPass(PassRegistry&);
+void initializeTypeBasedAAWrapperPassPass(PassRegistry &);
+void initializeTypePromotionLegacyPass(PassRegistry &);
 void initializeInitUndefPass(PassRegistry &);
 void initializeUniformityInfoWrapperPassPass(PassRegistry &);
 void initializeUnifyLoopExitsLegacyPassPass(PassRegistry &);
-void initializeUnpackMachineBundlesPass(PassRegistry&);
-void initializeUnreachableBlockElimLegacyPassPass(PassRegistry&);
-void initializeUnreachableMachineBlockElimPass(PassRegistry&);
-void initializeVerifierLegacyPassPass(PassRegistry&);
-void initializeVirtRegMapPass(PassRegistry&);
-void initializeVirtRegRewriterPass(PassRegistry&);
-void initializeWasmEHPreparePass(PassRegistry&);
-void initializeWinEHPreparePass(PassRegistry&);
-void initializeWriteBitcodePassPass(PassRegistry&);
-void initializeXRayInstrumentationPass(PassRegistry&);
+void initializeUnpackMachineBundlesPass(PassRegistry &);
+void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
+void initializeUnreachableMachineBlockElimPass(PassRegistry &);
+void initializeVerifierLegacyPassPass(PassRegistry &);
+void initializeVirtRegMapPass(PassRegistry &);
+void initializeVirtRegRewriterPass(PassRegistry &);
+void initializeWasmEHPreparePass(PassRegistry &);
+void initializeWinEHPreparePass(PassRegistry &);
+void initializeWriteBitcodePassPass(PassRegistry &);
+void initializeXRayInstrumentationPass(PassRegistry &);
 
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 967238e8d03b6e0..1da02153d846f11 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -48,110 +48,112 @@
 #include <cstdlib>
 
 namespace {
-  struct ForcePassLinking {
-    ForcePassLinking() {
-      // We must reference the passes in such a way that compilers will not
-      // delete it all as dead code, even with whole program optimization,
-      // yet is effectively a NO-OP. As the compiler isn't smart enough
-      // to know that getenv() never returns -1, this will do the job.
-      // This is so that globals in the translation units where these functions
-      // are defined are forced to be initialized, populating various
-      // registries.
-      if (std::getenv("bar") != (char*) -1)
-        return;
+struct ForcePassLinking {
+  ForcePassLinking() {
+    // We must reference the passes in such a way that compilers will not
+    // delete it all as dead code, even with whole program optimization,
+    // yet is effectively a NO-OP. As the compiler isn't smart enough
+    // to know that getenv() never returns -1, this will do the job.
+    // This is so that globals in the translation units where these functions
+    // are defined are forced to be initialized, populating various
+    // registries.
+    if (std::getenv("bar") != (char *)-1)
+      return;
 
-      (void)llvm::createAtomicExpandLegacyPass();
-      (void) llvm::createBasicAAWrapperPass();
-      (void) llvm::createSCEVAAWrapperPass();
-      (void) llvm::createTypeBasedAAWrapperPass();
-      (void) llvm::createScopedNoAliasAAWrapperPass();
-      (void) llvm::createBreakCriticalEdgesPass();
-      (void) llvm::createCallGraphDOTPrinterPass();
-      (void) llvm::createCallGraphViewerPass();
-      (void) llvm::createCFGSimplificationPass();
-      (void) llvm::createStructurizeCFGPass();
-      (void) llvm::createDXILResourceWrapperPassPass();
-      (void) llvm::createDeadArgEliminationPass();
-      (void) llvm::createDeadCodeEliminationPass();
-      (void) llvm::createDependenceAnalysisWrapperPass();
-      (void) llvm::createDomOnlyPrinterWrapperPassPass();
-      (void) llvm::createDomPrinterWrapperPassPass();
-      (void) llvm::createDomOnlyViewerWrapperPassPass();
-      (void) llvm::createDomViewerWrapperPassPass();
-      (void) llvm::createAlwaysInlinerLegacyPass();
-      (void) llvm::createGlobalsAAWrapperPass();
-      (void) llvm::createInstSimplifyLegacyPass();
-      (void) llvm::createInstructionCombiningPass();
-      (void) llvm::createJMCInstrumenterPass();
-      (void) llvm::createKCFIPass();
-      (void) llvm::createLCSSAPass();
-      (void) llvm::createLICMPass();
-      (void) llvm::createLazyValueInfoPass();
-      (void) llvm::createLoopExtractorPass();
-      (void) llvm::createLoopSimplifyPass();
-      (void) llvm::createLoopStrengthReducePass();
-      (void)llvm::createLoopUnrollPass();
-      (void) llvm::createLowerGlobalDtorsLegacyPass();
-      (void) llvm::createLowerInvokePass();
-      (void) llvm::createLowerSwitchPass();
-      (void) llvm::createNaryReassociatePass();
-      (void) llvm::createObjCARCContractPass();
-      (void) llvm::createPromoteMemoryToRegisterPass();
-      (void)llvm::createPostDomOnlyPrinterWrapperPassPass();
-      (void)llvm::createPostDomPrinterWrapperPassPass();
-      (void)llvm::createPostDomOnlyViewerWrapperPassPass();
-      (void)llvm::createPostDomViewerWrapperPassPass();
-      (void) llvm::createReassociatePass();
-      (void) llvm::createRegionInfoPass();
-      (void) llvm::createRegionOnlyPrinterPass();
-      (void) llvm::createRegionOnlyViewerPass();
-      (void) llvm::createRegionPrinterPass();
-      (void) llvm::createRegionViewerPass();
-      (void) llvm::createSafeStackPass();
-      (void) llvm::createSROAPass();
-      (void) llvm::createSingleLoopExtractorPass();
-      (void) llvm::createTailCallEliminationPass();
-      (void)llvm::createTLSVariableHoistPass();
-      (void) llvm::createConstantHoistingPass();
-      (void)llvm::createCodeGenPrepareLegacyPass();
-      (void) llvm::createPostInlineEntryExitInstrumenterPass();
-      (void) llvm::createEarlyCSEPass();
-      (void) llvm::createGVNPass();
-      (void) llvm::createPostDomTree();
-      (void) llvm::createMergeICmpsLegacyPass();
-      (void) llvm::createExpandLargeDivRemPass();
-      (void)llvm::createExpandMemCmpLegacyPass();
-      std::string buf;
-      llvm::raw_string_ostream os(buf);
-      (void) llvm::createPrintModulePass(os);
-      (void) llvm::createPrintFunctionPass(os);
-      (void) llvm::createSinkingPass();
-      (void) llvm::createLowerAtomicPass();
-      (void) llvm::createLoadStoreVectorizerPass();
-      (void) llvm::createPartiallyInlineLibCallsPass();
-      (void) llvm::createSeparateConstOffsetFromGEPPass();
-      (void) llvm::createSpeculativeExecutionPass();
-      (void) llvm::createSpeculativeExecutionIfHasBranchDivergencePass();
-      (void) llvm::createStraightLineStrengthReducePass();
-      (void)llvm::createScalarizeMaskedMemIntrinLegacyPass();
-      (void) llvm::createHardwareLoopsLegacyPass();
-      (void) llvm::createUnifyLoopExitsPass();
-      (void) llvm::createFixIrreduciblePass();
-      (void)llvm::createSelectOptimizePass();
+    (void)llvm::createAtomicExpandLegacyPass();
+    (void)llvm::createBasicAAWrapperPass();
+    (void)llvm::createSCEVAAWrapperPass();
+    (void)llvm::createTypeBasedAAWrapperPass();
+    (void)llvm::createScopedNoAliasAAWrapperPass();
+    (void)llvm::createBreakCriticalEdgesPass();
+    (void)llvm::createCallGraphDOTPrinterPass();
+    (void)llvm::createCallGraphViewerPass();
+    (void)llvm::createCFGSimplificationPass();
+    (void)llvm::createStructurizeCFGPass();
+    (void)llvm::createDXILResourceWrapperPassPass();
+    (void)llvm::createDeadArgEliminationPass();
+    (void)llvm::createDeadCodeEliminationPass();
+    (void)llvm::createDependenceAnalysisWrapperPass();
+    (void)llvm::createDomOnlyPrinterWrapperPassPass();
+    (void)llvm::createDomPrinterWrapperPassPass();
+    (void)llvm::createDomOnlyViewerWrapperPassPass();
+    (void)llvm::createDomViewerWrapperPassPass();
+    (void)llvm::createAlwaysInlinerLegacyPass();
+    (void)llvm::createGlobalsAAWrapperPass();
+    (void)llvm::createInstSimplifyLegacyPass();
+    (void)llvm::createInstructionCombiningPass();
+    (void)llvm::createJMCInstrumenterPass();
+    (void)llvm::createKCFIPass();
+    (void)llvm::createLCSSAPass();
+    (void)llvm::createLICMPass();
+    (void)llvm::createLazyValueInfoPass();
+    (void)llvm::createLoopExtractorPass();
+    (void)llvm::createLoopSimplifyPass();
+    (void)llvm::createLoopStrengthReducePass();
+    (void)llvm::createLoopTermFoldPass();
+    (void)llvm::createLoopUnrollPass();
+    (void)llvm::createLowerGlobalDtorsLegacyPass();
+    (void)llvm::createLowerInvokePass();
+    (void)llvm::createLowerSwitchPass();
+    (void)llvm::createNaryReassociatePass();
+    (void)llvm::createObjCARCContractPass();
+    (void)llvm::createPromoteMemoryToRegisterPass();
+    (void)llvm::createPostDomOnlyPrinterWrapperPassPass();
+    (void)llvm::createPostDomPrinterWrapperPassPass();
+    (void)llvm::createPostDomOnlyViewerWrapperPassPass();
+    (void)llvm::createPostDomViewerWrapperPassPass();
+    (void)llvm::createReassociatePass();
+    (void)llvm::createRegionInfoPass();
+    (void)llvm::createRegionOnlyPrinterPass();
+    (void)llvm::createRegionOnlyViewerPass();
+    (void)llvm::createRegionPrinterPass();
+    (void)llvm::createRegionViewerPass();
+    (void)llvm::createSafeStackPass();
+    (void)llvm::createSROAPass();
+    (void)llvm::createSingleLoopExtractorPass();
+    (void)llvm::createTailCallEliminationPass();
+    (void)llvm::createTLSVariableHoistPass();
+    (void)llvm::createConstantHoistingPass();
+    (void)llvm::createCodeGenPrepareLegacyPass();
+    (void)llvm::createPostInlineEntryExitInstrumenterPass();
+    (void)llvm::createEarlyCSEPass();
+    (void)llvm::createGVNPass();
+    (void)llvm::createPostDomTree();
+    (void)llvm::createMergeICmpsLegacyPass();
+    (void)llvm::createExpandLargeDivRemPass();
+    (void)llvm::createExpandMemCmpLegacyPass();
+    std::string buf;
+    llvm::raw_string_ostream os(buf);
+    (void)llvm::createPrintModulePass(os);
+    (void)llvm::createPrintFunctionPass(os);
+    (void)llvm::createSinkingPass();
+    (void)llvm::createLowerAtomicPass();
+    (void)llvm::createLoadStoreVectorizerPass();
+    (void)llvm::createPartiallyInlineLibCallsPass();
+    (void)llvm::createSeparateConstOffsetFromGEPPass();
+    (void)llvm::createSpeculativeExecutionPass();
+    (void)llvm::createSpeculativeExecutionIfHasBranchDivergencePass();
+    (void)llvm::createStraightLineStrengthReducePass();
+    (void)llvm::createScalarizeMaskedMemIntrinLegacyPass();
+    (void)llvm::createHardwareLoopsLegacyPass();
+    (void)llvm::createUnifyLoopExitsPass();
+    (void)llvm::createFixIrreduciblePass();
+    (void)llvm::createSelectOptimizePass();
 
-      (void)new llvm::ScalarEvolutionWrapperPass();
-      llvm::Function::Create(nullptr, llvm::GlobalValue::ExternalLinkage)->viewCFGOnly();
-      llvm::RGPassManager RGM;
-      llvm::TargetLibraryInfoImpl TLII;
-      llvm::TargetLibraryInfo TLI(TLII);
-      llvm::AliasAnalysis AA(TLI);
-      llvm::BatchAAResults BAA(AA);
-      llvm::AliasSetTracker X(BAA);
-      X.add(llvm::MemoryLocation()); // for -print-alias-sets
-      (void) llvm::AreStatisticsEnabled();
-      (void) llvm::sys::RunningOnValgrind();
-    }
-  } ForcePassLinking; // Force link by creating a global definition.
-}
+    (void)new llvm::ScalarEvolutionWrapperPass();
+    llvm::Function::Create(nullptr, llvm::GlobalValue::ExternalLinkage)
+        ->viewCFGOnly();
+    llvm::RGPassManager RGM;
+    llvm::TargetLibraryInfoImpl TLII;
+    llvm::TargetLibraryInfo TLI(TLII);
+    llvm::AliasAnalysis AA(TLI);
+    llvm::BatchAAResults BAA(AA);
+    llvm::AliasSetTracker X(BAA);
+    X.add(llvm::MemoryLocation()); // for -print-alias-sets
+    (void)llvm::AreStatisticsEnabled();
+    (void)llvm::sys::RunningOnValgrind();
+  }
+} ForcePassLinking; // Force link by creating a global definition.
+} // namespace
 
 #endif
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index add60ef7006b85a..ccb7c524d45f8d0 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -85,14 +85,6 @@ class MCAsmInfo {
   /// Default is false.
   bool HasSubsectionsViaSymbols = false;
 
-  /// True if this is a MachO target that supports the macho-specific .zerofill
-  /// directive for emitting BSS Symbols.  Default is false.
-  bool HasMachoZeroFillDirective = false;
-
-  /// True if this is a MachO target that supports the macho-specific .tbss
-  /// directive for emitting thread local BSS Symbols.  Default is false.
-  bool HasMachoTBSSDirective = false;
-
   /// True if this is a non-GNU COFF target. The COFF port of the GNU linker
   /// doesn't handle associative comdats in the way that we would like to use
   /// them.
@@ -348,21 +340,6 @@ class MCAsmInfo {
   ///   .long a
   bool SetDirectiveSuppressesReloc = false;
 
-  /// False if the assembler requires that we use
-  /// \code
-  ///   Lc = a - b
-  ///   .long Lc
-  /// \endcode
-  //
-  /// instead of
-  //
-  /// \code
-  ///   .long a - b
-  /// \endcode
-  ///
-  ///  Defaults to true.
-  bool HasAggressiveSymbolFolding = true;
-
   /// True is .comm's and .lcomms optional alignment is to be specified in bytes
   /// instead of log2(n).  Defaults to true.
   bool COMMDirectiveAlignmentIsInBytes = true;
@@ -405,10 +382,6 @@ class MCAsmInfo {
   /// to false.
   bool HasNoDeadStrip = false;
 
-  /// True if this target supports the MachO .alt_entry directive.  Defaults to
-  /// false.
-  bool HasAltEntry = false;
-
   /// Used to declare a global as being a weak symbol. Defaults to ".weak".
   const char *WeakDirective;
 
@@ -416,10 +389,6 @@ class MCAsmInfo {
   /// undefined symbol.  Defaults to nullptr.
   const char *WeakRefDirective = nullptr;
 
-  /// True if we have a directive to declare a global as being a weak defined
-  /// symbol.  Defaults to false.
-  bool HasWeakDefDirective = false;
-
   /// True if we have a directive to declare a global as being a weak defined
   /// symbol that can be hidden (unexported).  Defaults to false.
   bool HasWeakDefCanBeHiddenDirective = false;
@@ -524,6 +493,9 @@ class MCAsmInfo {
   /// Preserve Comments in assembly
   bool PreserveAsmComments;
 
+  /// The column (zero-based) at which asm comments should be printed.
+  unsigned CommentColumn = 40;
+
   /// True if the integrated assembler should interpret 'a >> b' constant
   /// expressions as logical rather than arithmetic.
   bool UseLogicalShr = true;
@@ -618,8 +590,7 @@ class MCAsmInfo {
 
   // Accessors.
 
-  bool hasMachoZeroFillDirective() const { return HasMachoZeroFillDirective; }
-  bool hasMachoTBSSDirective() const { return HasMachoTBSSDirective; }
+  bool isMachO() const { return HasSubsectionsViaSymbols; }
   bool hasCOFFAssociativeComdats() const { return HasCOFFAssociativeComdats; }
   bool hasCOFFComdatConstants() const { return HasCOFFComdatConstants; }
   bool hasVisibilityOnlyWithLinkage() const {
@@ -638,9 +609,8 @@ class MCAsmInfo {
   bool getStarIsPC() const { return StarIsPC; }
   const char *getSeparatorString() const { return SeparatorString; }
 
-  /// This indicates the column (zero-based) at which asm comments should be
-  /// printed.
-  unsigned getCommentColumn() const { return 40; }
+  unsigned getCommentColumn() const { return CommentColumn; }
+  void setCommentColumn(unsigned Col) { CommentColumn = Col; }
 
   StringRef getCommentString() const { return CommentString; }
   bool getRestrictCommentStringToStartOfStatement() const {
@@ -722,8 +692,6 @@ class MCAsmInfo {
     return SetDirectiveSuppressesReloc;
   }
 
-  bool hasAggressiveSymbolFolding() const { return HasAggressiveSymbolFolding; }
-
   bool getCOMMDirectiveAlignmentIsInBytes() const {
     return COMMDirectiveAlignmentIsInBytes;
   }
@@ -744,10 +712,8 @@ class MCAsmInfo {
   bool hasFourStringsDotFile() const { return HasFourStringsDotFile; }
   bool hasIdentDirective() const { return HasIdentDirective; }
   bool hasNoDeadStrip() const { return HasNoDeadStrip; }
-  bool hasAltEntry() const { return HasAltEntry; }
   const char *getWeakDirective() const { return WeakDirective; }
   const char *getWeakRefDirective() const { return WeakRefDirective; }
-  bool hasWeakDefDirective() const { return HasWeakDefDirective; }
 
   bool hasWeakDefCanBeHiddenDirective() const {
     return HasWeakDefCanBeHiddenDirective;
diff --git a/llvm/include/llvm/MC/MCELFObjectWriter.h b/llvm/include/llvm/MC/MCELFObjectWriter.h
index 3b418083eecceba..d17fc931d1561c1 100644
--- a/llvm/include/llvm/MC/MCELFObjectWriter.h
+++ b/llvm/include/llvm/MC/MCELFObjectWriter.h
@@ -38,18 +38,15 @@ struct ELFRelocationEntry {
   unsigned Type;   // The type of the relocation.
   uint64_t Addend; // The addend to use.
   const MCSymbolELF *OriginalSymbol; // The original value of Symbol if we changed it.
-  uint64_t OriginalAddend; // The original value of addend.
 
   ELFRelocationEntry(uint64_t Offset, const MCSymbolELF *Symbol, unsigned Type,
-                     uint64_t Addend, const MCSymbolELF *OriginalSymbol,
-                     uint64_t OriginalAddend)
+                     uint64_t Addend, const MCSymbolELF *OriginalSymbol)
       : Offset(Offset), Symbol(Symbol), Type(Type), Addend(Addend),
-        OriginalSymbol(OriginalSymbol), OriginalAddend(OriginalAddend) {}
+        OriginalSymbol(OriginalSymbol) {}
 
   void print(raw_ostream &Out) const {
     Out << "Off=" << Offset << ", Sym=" << Symbol << ", Type=" << Type
-        << ", Addend=" << Addend << ", OriginalSymbol=" << OriginalSymbol
-        << ", OriginalAddend=" << OriginalAddend;
+        << ", Addend=" << Addend << ", OriginalSymbol=" << OriginalSymbol;
   }
 
   LLVM_DUMP_METHOD void dump() const { print(errs()); }
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 8e669ee5791239d..05baf514fa72104 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -79,6 +79,7 @@ FUNCTION_PASS("win-eh-prepare", WinEHPreparePass())
 #define LOOP_PASS(NAME, CREATE_PASS)
 #endif
 LOOP_PASS("loop-reduce", LoopStrengthReducePass())
+LOOP_PASS("loop-term-fold", LoopTermFoldPass())
 #undef LOOP_PASS
 
 #ifndef MACHINE_MODULE_PASS
diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index a2e2a32e9c01eb9..a881bdf28f22c25 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -107,6 +107,7 @@ namespace llvm {
 namespace sandboxir {
 
 class BasicBlock;
+class ConstantInt;
 class Context;
 class Function;
 class Instruction;
@@ -130,6 +131,10 @@ class CastInst;
 class PtrToIntInst;
 class BitCastInst;
 class AllocaInst;
+class SwitchInst;
+class UnaryOperator;
+class BinaryOperator;
+class AtomicRMWInst;
 class AtomicCmpXchgInst;
 
 /// Iterator for the `Use` edges of a User's operands.
@@ -249,6 +254,10 @@ class Value {
   friend class InvokeInst;         // For getting `Val`.
   friend class CallBrInst;         // For getting `Val`.
   friend class GetElementPtrInst;  // For getting `Val`.
+  friend class SwitchInst;         // For getting `Val`.
+  friend class UnaryOperator;      // For getting `Val`.
+  friend class BinaryOperator;     // For getting `Val`.
+  friend class AtomicRMWInst;      // For getting `Val`.
   friend class AtomicCmpXchgInst;  // For getting `Val`.
   friend class AllocaInst;         // For getting `Val`.
   friend class CastInst;           // For getting `Val`.
@@ -483,22 +492,22 @@ class Constant : public sandboxir::User {
       : sandboxir::User(ClassID::Constant, C, SBCtx) {}
   Constant(ClassID ID, llvm::Constant *C, sandboxir::Context &SBCtx)
       : sandboxir::User(ID, C, SBCtx) {}
-  friend class Function; // For constructor
-  friend class Context;  // For constructor.
-  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+  friend class ConstantInt; // For constructor.
+  friend class Function;    // For constructor
+  friend class Context;     // For constructor.
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const override {
     return getOperandUseDefault(OpIdx, Verify);
   }
 
 public:
-  static Constant *createInt(Type *Ty, uint64_t V, Context &Ctx,
-                             bool IsSigned = false);
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
     return From->getSubclassID() == ClassID::Constant ||
+           From->getSubclassID() == ClassID::ConstantInt ||
            From->getSubclassID() == ClassID::Function;
   }
   sandboxir::Context &getParent() const { return getContext(); }
-  unsigned getUseOperandNo(const Use &Use) const final {
+  unsigned getUseOperandNo(const Use &Use) const override {
     return getUseOperandNoDefault(Use);
   }
 #ifndef NDEBUG
@@ -509,6 +518,41 @@ class Constant : public sandboxir::User {
 #endif
 };
 
+class ConstantInt : public Constant {
+  ConstantInt(llvm::ConstantInt *C, sandboxir::Context &Ctx)
+      : Constant(ClassID::ConstantInt, C, Ctx) {}
+  friend class Context; // For constructor.
+
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+    llvm_unreachable("ConstantInt has no operands!");
+  }
+
+public:
+  /// If Ty is a vector type, return a Constant with a splat of the given
+  /// value. Otherwise return a ConstantInt for the given value.
+  static ConstantInt *get(Type *Ty, uint64_t V, Context &Ctx,
+                          bool IsSigned = false);
+
+  // TODO: Implement missing functions.
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantInt;
+  }
+  unsigned getUseOperandNo(const Use &Use) const override {
+    llvm_unreachable("ConstantInt has no operands!");
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::ConstantInt>(Val) && "Expected a ConstantInst!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
 /// Iterator for `Instruction`s in a `BasicBlock.
 /// \Returns an sandboxir::Instruction & when derereferenced.
 class BBIterator {
@@ -630,6 +674,10 @@ class Instruction : public sandboxir::User {
   friend class InvokeInst;         // For getTopmostLLVMInstruction().
   friend class CallBrInst;         // For getTopmostLLVMInstruction().
   friend class GetElementPtrInst;  // For getTopmostLLVMInstruction().
+  friend class SwitchInst;         // For getTopmostLLVMInstruction().
+  friend class UnaryOperator;      // For getTopmostLLVMInstruction().
+  friend class BinaryOperator;     // For getTopmostLLVMInstruction().
+  friend class AtomicRMWInst;      // For getTopmostLLVMInstruction().
   friend class AtomicCmpXchgInst;  // For getTopmostLLVMInstruction().
   friend class AllocaInst;         // For getTopmostLLVMInstruction().
   friend class CastInst;           // For getTopmostLLVMInstruction().
@@ -1432,6 +1480,283 @@ class GetElementPtrInst final
   // TODO: Add missing member functions.
 };
 
+class SwitchInst : public SingleLLVMInstructionImpl<llvm::SwitchInst> {
+public:
+  SwitchInst(llvm::SwitchInst *SI, Context &Ctx)
+      : SingleLLVMInstructionImpl(ClassID::Switch, Opcode::Switch, SI, Ctx) {}
+
+  static constexpr const unsigned DefaultPseudoIndex =
+      llvm::SwitchInst::DefaultPseudoIndex;
+
+  static SwitchInst *create(Value *V, BasicBlock *Dest, unsigned NumCases,
+                            BasicBlock::iterator WhereIt, BasicBlock *WhereBB,
+                            Context &Ctx, const Twine &Name = "");
+
+  Value *getCondition() const;
+  void setCondition(Value *V);
+  BasicBlock *getDefaultDest() const;
+  bool defaultDestUndefined() const {
+    return cast<llvm::SwitchInst>(Val)->defaultDestUndefined();
+  }
+  void setDefaultDest(BasicBlock *DefaultCase);
+  unsigned getNumCases() const {
+    return cast<llvm::SwitchInst>(Val)->getNumCases();
+  }
+
+  using CaseHandle =
+      llvm::SwitchInst::CaseHandleImpl<SwitchInst, ConstantInt, BasicBlock>;
+  using ConstCaseHandle =
+      llvm::SwitchInst::CaseHandleImpl<const SwitchInst, const ConstantInt,
+                                       const BasicBlock>;
+  using CaseIt = llvm::SwitchInst::CaseIteratorImpl<CaseHandle>;
+  using ConstCaseIt = llvm::SwitchInst::CaseIteratorImpl<ConstCaseHandle>;
+
+  /// Returns a read/write iterator that points to the first case in the
+  /// SwitchInst.
+  CaseIt case_begin() { return CaseIt(this, 0); }
+  ConstCaseIt case_begin() const { return ConstCaseIt(this, 0); }
+  /// Returns a read/write iterator that points one past the last in the
+  /// SwitchInst.
+  CaseIt case_end() { return CaseIt(this, getNumCases()); }
+  ConstCaseIt case_end() const { return ConstCaseIt(this, getNumCases()); }
+  /// Iteration adapter for range-for loops.
+  iterator_range<CaseIt> cases() {
+    return make_range(case_begin(), case_end());
+  }
+  iterator_range<ConstCaseIt> cases() const {
+    return make_range(case_begin(), case_end());
+  }
+  CaseIt case_default() { return CaseIt(this, DefaultPseudoIndex); }
+  ConstCaseIt case_default() const {
+    return ConstCaseIt(this, DefaultPseudoIndex);
+  }
+  CaseIt findCaseValue(const ConstantInt *C) {
+    return CaseIt(
+        this,
+        const_cast<const SwitchInst *>(this)->findCaseValue(C)->getCaseIndex());
+  }
+  ConstCaseIt findCaseValue(const ConstantInt *C) const {
+    ConstCaseIt I = llvm::find_if(cases(), [C](const ConstCaseHandle &Case) {
+      return Case.getCaseValue() == C;
+    });
+    if (I != case_end())
+      return I;
+    return case_default();
+  }
+  ConstantInt *findCaseDest(BasicBlock *BB);
+
+  void addCase(ConstantInt *OnVal, BasicBlock *Dest);
+  /// This method removes the specified case and its successor from the switch
+  /// instruction. Note that this operation may reorder the remaining cases at
+  /// index idx and above.
+  /// Note:
+  /// This action invalidates iterators for all cases following the one removed,
+  /// including the case_end() iterator. It returns an iterator for the next
+  /// case.
+  CaseIt removeCase(CaseIt It);
+
+  unsigned getNumSuccessors() const {
+    return cast<llvm::SwitchInst>(Val)->getNumSuccessors();
+  }
+  BasicBlock *getSuccessor(unsigned Idx) const;
+  void setSuccessor(unsigned Idx, BasicBlock *NewSucc);
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::Switch;
+  }
+};
+
+class UnaryOperator : public UnaryInstruction {
+  static Opcode getUnaryOpcode(llvm::Instruction::UnaryOps UnOp) {
+    switch (UnOp) {
+    case llvm::Instruction::FNeg:
+      return Opcode::FNeg;
+    case llvm::Instruction::UnaryOpsEnd:
+      llvm_unreachable("Bad UnOp!");
+    }
+    llvm_unreachable("Unhandled UnOp!");
+  }
+  UnaryOperator(llvm::UnaryOperator *UO, Context &Ctx)
+      : UnaryInstruction(ClassID::UnOp, getUnaryOpcode(UO->getOpcode()), UO,
+                         Ctx) {}
+  friend Context; // for constructor.
+public:
+  static Value *create(Instruction::Opcode Op, Value *OpV, BBIterator WhereIt,
+                       BasicBlock *WhereBB, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *create(Instruction::Opcode Op, Value *OpV,
+                       Instruction *InsertBefore, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *create(Instruction::Opcode Op, Value *OpV,
+                       BasicBlock *InsertAtEnd, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                      Value *CopyFrom, BBIterator WhereIt,
+                                      BasicBlock *WhereBB, Context &Ctx,
+                                      const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                      Value *CopyFrom,
+                                      Instruction *InsertBefore, Context &Ctx,
+                                      const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                      Value *CopyFrom, BasicBlock *InsertAtEnd,
+                                      Context &Ctx, const Twine &Name = "");
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::UnOp;
+  }
+};
+
+class BinaryOperator : public SingleLLVMInstructionImpl<llvm::BinaryOperator> {
+  static Opcode getBinOpOpcode(llvm::Instruction::BinaryOps BinOp) {
+    switch (BinOp) {
+    case llvm::Instruction::Add:
+      return Opcode::Add;
+    case llvm::Instruction::FAdd:
+      return Opcode::FAdd;
+    case llvm::Instruction::Sub:
+      return Opcode::Sub;
+    case llvm::Instruction::FSub:
+      return Opcode::FSub;
+    case llvm::Instruction::Mul:
+      return Opcode::Mul;
+    case llvm::Instruction::FMul:
+      return Opcode::FMul;
+    case llvm::Instruction::UDiv:
+      return Opcode::UDiv;
+    case llvm::Instruction::SDiv:
+      return Opcode::SDiv;
+    case llvm::Instruction::FDiv:
+      return Opcode::FDiv;
+    case llvm::Instruction::URem:
+      return Opcode::URem;
+    case llvm::Instruction::SRem:
+      return Opcode::SRem;
+    case llvm::Instruction::FRem:
+      return Opcode::FRem;
+    case llvm::Instruction::Shl:
+      return Opcode::Shl;
+    case llvm::Instruction::LShr:
+      return Opcode::LShr;
+    case llvm::Instruction::AShr:
+      return Opcode::AShr;
+    case llvm::Instruction::And:
+      return Opcode::And;
+    case llvm::Instruction::Or:
+      return Opcode::Or;
+    case llvm::Instruction::Xor:
+      return Opcode::Xor;
+    case llvm::Instruction::BinaryOpsEnd:
+      llvm_unreachable("Bad BinOp!");
+    }
+    llvm_unreachable("Unhandled BinOp!");
+  }
+  BinaryOperator(llvm::BinaryOperator *BinOp, Context &Ctx)
+      : SingleLLVMInstructionImpl(ClassID::BinaryOperator,
+                                  getBinOpOpcode(BinOp->getOpcode()), BinOp,
+                                  Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  static Value *create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                       BBIterator WhereIt, BasicBlock *WhereBB, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                       Instruction *InsertBefore, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                       BasicBlock *InsertAtEnd, Context &Ctx,
+                       const Twine &Name = "");
+
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                      Value *RHS, Value *CopyFrom,
+                                      BBIterator WhereIt, BasicBlock *WhereBB,
+                                      Context &Ctx, const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                      Value *RHS, Value *CopyFrom,
+                                      Instruction *InsertBefore, Context &Ctx,
+                                      const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                      Value *RHS, Value *CopyFrom,
+                                      BasicBlock *InsertAtEnd, Context &Ctx,
+                                      const Twine &Name = "");
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::BinaryOperator;
+  }
+  void swapOperands() { swapOperandsInternal(0, 1); }
+};
+
+class AtomicRMWInst : public SingleLLVMInstructionImpl<llvm::AtomicRMWInst> {
+  AtomicRMWInst(llvm::AtomicRMWInst *Atomic, Context &Ctx)
+      : SingleLLVMInstructionImpl(ClassID::AtomicRMW,
+                                  Instruction::Opcode::AtomicRMW, Atomic, Ctx) {
+  }
+  friend class Context; // For constructor.
+
+public:
+  using BinOp = llvm::AtomicRMWInst::BinOp;
+  BinOp getOperation() const {
+    return cast<llvm::AtomicRMWInst>(Val)->getOperation();
+  }
+  static StringRef getOperationName(BinOp Op) {
+    return llvm::AtomicRMWInst::getOperationName(Op);
+  }
+  static bool isFPOperation(BinOp Op) {
+    return llvm::AtomicRMWInst::isFPOperation(Op);
+  }
+  void setOperation(BinOp Op) {
+    cast<llvm::AtomicRMWInst>(Val)->setOperation(Op);
+  }
+  Align getAlign() const { return cast<llvm::AtomicRMWInst>(Val)->getAlign(); }
+  void setAlignment(Align Align);
+  bool isVolatile() const {
+    return cast<llvm::AtomicRMWInst>(Val)->isVolatile();
+  }
+  void setVolatile(bool V);
+  AtomicOrdering getOrdering() const {
+    return cast<llvm::AtomicRMWInst>(Val)->getOrdering();
+  }
+  void setOrdering(AtomicOrdering Ordering);
+  SyncScope::ID getSyncScopeID() const {
+    return cast<llvm::AtomicRMWInst>(Val)->getSyncScopeID();
+  }
+  void setSyncScopeID(SyncScope::ID SSID);
+  Value *getPointerOperand();
+  const Value *getPointerOperand() const {
+    return const_cast<AtomicRMWInst *>(this)->getPointerOperand();
+  }
+  Value *getValOperand();
+  const Value *getValOperand() const {
+    return const_cast<AtomicRMWInst *>(this)->getValOperand();
+  }
+  unsigned getPointerAddressSpace() const {
+    return cast<llvm::AtomicRMWInst>(Val)->getPointerAddressSpace();
+  }
+  bool isFloatingPointOperation() const {
+    return cast<llvm::AtomicRMWInst>(Val)->isFloatingPointOperation();
+  }
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::AtomicRMW;
+  }
+
+  static AtomicRMWInst *create(BinOp Op, Value *Ptr, Value *Val,
+                               MaybeAlign Align, AtomicOrdering Ordering,
+                               BBIterator WhereIt, BasicBlock *WhereBB,
+                               Context &Ctx,
+                               SyncScope::ID SSID = SyncScope::System,
+                               const Twine &Name = "");
+  static AtomicRMWInst *create(BinOp Op, Value *Ptr, Value *Val,
+                               MaybeAlign Align, AtomicOrdering Ordering,
+                               Instruction *InsertBefore, Context &Ctx,
+                               SyncScope::ID SSID = SyncScope::System,
+                               const Twine &Name = "");
+  static AtomicRMWInst *create(BinOp Op, Value *Ptr, Value *Val,
+                               MaybeAlign Align, AtomicOrdering Ordering,
+                               BasicBlock *InsertAtEnd, Context &Ctx,
+                               SyncScope::ID SSID = SyncScope::System,
+                               const Twine &Name = "");
+};
+
 class AtomicCmpXchgInst
     : public SingleLLVMInstructionImpl<llvm::AtomicCmpXchgInst> {
   AtomicCmpXchgInst(llvm::AtomicCmpXchgInst *Atomic, Context &Ctx)
@@ -1844,7 +2169,7 @@ class Context {
   Constant *getOrCreateConstant(llvm::Constant *LLVMC) {
     return cast<Constant>(getOrCreateValueInternal(LLVMC, 0));
   }
-  friend class Constant; // For getOrCreateConstant().
+  friend class ConstantInt; // For getOrCreateConstant().
   /// Create a sandboxir::BasicBlock for an existing LLVM IR \p BB. This will
   /// also create all contents of the block.
   BasicBlock *createBasicBlock(llvm::BasicBlock *BB);
@@ -1876,6 +2201,14 @@ class Context {
   friend CallBrInst; // For createCallBrInst()
   GetElementPtrInst *createGetElementPtrInst(llvm::GetElementPtrInst *I);
   friend GetElementPtrInst; // For createGetElementPtrInst()
+  SwitchInst *createSwitchInst(llvm::SwitchInst *I);
+  friend SwitchInst; // For createSwitchInst()
+  UnaryOperator *createUnaryOperator(llvm::UnaryOperator *I);
+  friend UnaryOperator; // For createUnaryOperator()
+  BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
+  friend BinaryOperator; // For createBinaryOperator()
+  AtomicRMWInst *createAtomicRMWInst(llvm::AtomicRMWInst *I);
+  friend AtomicRMWInst; // For createAtomicRMWInst()
   AtomicCmpXchgInst *createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I);
   friend AtomicCmpXchgInst; // For createAtomicCmpXchgInst()
   AllocaInst *createAllocaInst(llvm::AllocaInst *I);
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index 114dc8505ecd6f9..2b9b44c529b30d8 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -19,6 +19,7 @@ DEF_VALUE(Argument, Argument)
 DEF_USER(User, User)
 DEF_VALUE(Block, BasicBlock)
 DEF_USER(Constant, Constant)
+DEF_USER(ConstantInt, ConstantInt)
 
 #ifndef DEF_INSTR
 #define DEF_INSTR(ID, OPCODE, CLASS)
@@ -32,19 +33,44 @@ DEF_USER(Constant, Constant)
 #define OPCODES(...)
 #endif
 // clang-format off
-//        ClassID,        Opcode(s),          Class
-DEF_INSTR(Opaque,         OP(Opaque),         OpaqueInst)
+//       ClassID,        Opcode(s),         Class
+DEF_INSTR(Opaque,        OP(Opaque),        OpaqueInst)
 DEF_INSTR(ExtractElement, OP(ExtractElement), ExtractElementInst)
-DEF_INSTR(InsertElement,  OP(InsertElement),  InsertElementInst)
-DEF_INSTR(Select,         OP(Select),         SelectInst)
-DEF_INSTR(Br,             OP(Br),             BranchInst)
-DEF_INSTR(Load,           OP(Load),           LoadInst)
-DEF_INSTR(Store,          OP(Store),          StoreInst)
-DEF_INSTR(Ret,            OP(Ret),            ReturnInst)
-DEF_INSTR(Call,           OP(Call),           CallInst)
-DEF_INSTR(Invoke,         OP(Invoke),         InvokeInst)
-DEF_INSTR(CallBr,         OP(CallBr),         CallBrInst)
-DEF_INSTR(GetElementPtr,  OP(GetElementPtr),  GetElementPtrInst)
+DEF_INSTR(InsertElement, OP(InsertElement), InsertElementInst)
+DEF_INSTR(Select,        OP(Select),        SelectInst)
+DEF_INSTR(Br,            OP(Br),            BranchInst)
+DEF_INSTR(Load,          OP(Load),          LoadInst)
+DEF_INSTR(Store,         OP(Store),         StoreInst)
+DEF_INSTR(Ret,           OP(Ret),           ReturnInst)
+DEF_INSTR(Call,          OP(Call),          CallInst)
+DEF_INSTR(Invoke,        OP(Invoke),        InvokeInst)
+DEF_INSTR(CallBr,        OP(CallBr),        CallBrInst)
+DEF_INSTR(GetElementPtr, OP(GetElementPtr), GetElementPtrInst)
+DEF_INSTR(Switch,        OP(Switch),        SwitchInst)
+DEF_INSTR(UnOp,          OPCODES( \
+                         OP(FNeg) \
+                         ),                 UnaryOperator)
+DEF_INSTR(BinaryOperator, OPCODES(\
+                         OP(Add)  \
+                         OP(FAdd) \
+                         OP(Sub)  \
+                         OP(FSub) \
+                         OP(Mul)  \
+                         OP(FMul) \
+                         OP(UDiv) \
+                         OP(SDiv) \
+                         OP(FDiv) \
+                         OP(URem) \
+                         OP(SRem) \
+                         OP(FRem) \
+                         OP(Shl)  \
+                         OP(LShr) \
+                         OP(AShr) \
+                         OP(And)  \
+                         OP(Or)   \
+                         OP(Xor)  \
+                         ),                 BinaryOperator)
+DEF_INSTR(AtomicRMW,     OP(AtomicRMW),     AtomicRMWInst)
 DEF_INSTR(AtomicCmpXchg, OP(AtomicCmpXchg), AtomicCmpXchgInst)
 DEF_INSTR(Alloca,         OP(Alloca),         AllocaInst)
 DEF_INSTR(Cast,   OPCODES(\
diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index 80efbf9e8f6e082..22d910e1d73e728 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -59,6 +59,8 @@ class StoreInst;
 class Instruction;
 class Tracker;
 class AllocaInst;
+class SwitchInst;
+class ConstantInt;
 
 /// The base class for IR Change classes.
 class IRChangeBase {
@@ -96,25 +98,6 @@ class UseSet : public IRChangeBase {
 #endif
 };
 
-class PHISetIncoming : public IRChangeBase {
-  PHINode *PHI;
-  unsigned Idx;
-  PointerUnion<Value *, BasicBlock *> OrigValueOrBB;
-
-public:
-  enum class What {
-    Value,
-    Block,
-  };
-  PHISetIncoming(PHINode *PHI, unsigned Idx, What What);
-  void revert(Tracker &Tracker) final;
-  void accept() final {}
-#ifndef NDEBUG
-  void dump(raw_ostream &OS) const final { OS << "PHISetIncoming"; }
-  LLVM_DUMP_METHOD void dump() const final;
-#endif
-};
-
 class PHIRemoveIncoming : public IRChangeBase {
   PHINode *PHI;
   unsigned RemovedIdx;
@@ -250,19 +233,65 @@ class GenericSetter final : public IRChangeBase {
 #endif
 };
 
-class CallBrInstSetIndirectDest : public IRChangeBase {
-  CallBrInst *CallBr;
+/// Similar to GenericSetter but the setters/getters have an index as their
+/// first argument. This is commont in cases like: getOperand(unsigned Idx)
+template <auto GetterFn, auto SetterFn>
+class GenericSetterWithIdx final : public IRChangeBase {
+  /// Helper for getting the class type from the getter
+  template <typename ClassT, typename RetT>
+  static ClassT getClassTypeFromGetter(RetT (ClassT::*Fn)(unsigned) const);
+  template <typename ClassT, typename RetT>
+  static ClassT getClassTypeFromGetter(RetT (ClassT::*Fn)(unsigned));
+
+  using InstrT = decltype(getClassTypeFromGetter(GetterFn));
+  using SavedValT = std::invoke_result_t<decltype(GetterFn), InstrT, unsigned>;
+  InstrT *I;
+  SavedValT OrigVal;
   unsigned Idx;
-  BasicBlock *OrigIndirectDest;
 
 public:
-  CallBrInstSetIndirectDest(CallBrInst *CallBr, unsigned Idx);
+  GenericSetterWithIdx(InstrT *I, unsigned Idx)
+      : I(I), OrigVal((I->*GetterFn)(Idx)), Idx(Idx) {}
+  void revert(Tracker &Tracker) final { (I->*SetterFn)(Idx, OrigVal); }
+  void accept() final {}
+#ifndef NDEBUG
+  void dump(raw_ostream &OS) const final { OS << "GenericSetterWithIdx"; }
+  LLVM_DUMP_METHOD void dump() const final {
+    dump(dbgs());
+    dbgs() << "\n";
+  }
+#endif
+};
+
+class SwitchAddCase : public IRChangeBase {
+  SwitchInst *Switch;
+  ConstantInt *Val;
+
+public:
+  SwitchAddCase(SwitchInst *Switch, ConstantInt *Val)
+      : Switch(Switch), Val(Val) {}
   void revert(Tracker &Tracker) final;
   void accept() final {}
 #ifndef NDEBUG
-  void dump(raw_ostream &OS) const final { OS << "CallBrInstSetIndirectDest"; }
+  void dump(raw_ostream &OS) const final { OS << "SwitchAddCase"; }
   LLVM_DUMP_METHOD void dump() const final;
-#endif
+#endif // NDEBUG
+};
+
+class SwitchRemoveCase : public IRChangeBase {
+  SwitchInst *Switch;
+  ConstantInt *Val;
+  BasicBlock *Dest;
+
+public:
+  SwitchRemoveCase(SwitchInst *Switch, ConstantInt *Val, BasicBlock *Dest)
+      : Switch(Switch), Val(Val), Dest(Dest) {}
+  void revert(Tracker &Tracker) final;
+  void accept() final {}
+#ifndef NDEBUG
+  void dump(raw_ostream &OS) const final { OS << "SwitchRemoveCase"; }
+  LLVM_DUMP_METHOD void dump() const final;
+#endif // NDEBUG
 };
 
 class MoveInstr : public IRChangeBase {
diff --git a/llvm/include/llvm/Support/DXILABI.h b/llvm/include/llvm/Support/DXILABI.h
index a2222eec09ba85a..cf2c42c689889d8 100644
--- a/llvm/include/llvm/Support/DXILABI.h
+++ b/llvm/include/llvm/Support/DXILABI.h
@@ -22,23 +22,6 @@
 namespace llvm {
 namespace dxil {
 
-enum class ParameterKind : uint8_t {
-  Invalid = 0,
-  Void,
-  Half,
-  Float,
-  Double,
-  I1,
-  I8,
-  I16,
-  I32,
-  I64,
-  Overload,
-  CBufferRet,
-  ResourceRet,
-  DXILHandle,
-};
-
 enum class ResourceClass : uint8_t {
   SRV = 0,
   UAV,
diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index e568e42afcf4d20..a52a9f07bacd4ba 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -497,13 +497,21 @@ constexpr uint64_t alignTo(uint64_t Value, uint64_t Align) {
   return CeilDiv * Align;
 }
 
+/// Will overflow only if result is not representable in T.
+template <typename U, typename V, typename T = common_uint<U, V>>
+constexpr T alignToPowerOf2(U Value, V Align) {
+  assert(Align != 0 && (Align & (Align - 1)) == 0 &&
+         "Align must be a power of 2");
+  T NegAlign = static_cast<T>(0) - Align;
+  return (Value + (Align - 1)) & NegAlign;
+}
+
+/// Fallback when arguments aren't integral.
 constexpr uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align) {
   assert(Align != 0 && (Align & (Align - 1)) == 0 &&
          "Align must be a power of 2");
-  // Replace unary minus to avoid compilation error on Windows:
-  // "unary minus operator applied to unsigned type, result still unsigned"
-  uint64_t NegAlign = (~Align) + 1;
-  return (Value + Align - 1) & NegAlign;
+  uint64_t NegAlign = 0 - Align;
+  return (Value + (Align - 1)) & NegAlign;
 }
 
 /// If non-zero \p Skew is specified, the return value will be a minimal integer
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index c76596ce3e4f2d5..814c5e789cb3749 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -767,15 +767,6 @@ def zext_trunc_fold: GICombineRule <
   (apply [{ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }])
 >;
 
-// Fold ([asz]ext ([asz]ext x)) -> ([asz]ext x).
-def ext_ext_fold_matchinfo : GIDefMatchData<"std::tuple<Register, unsigned>">;
-def ext_ext_fold: GICombineRule <
-  (defs root:$root, ext_ext_fold_matchinfo:$matchinfo),
-  (match (wip_match_opcode G_ANYEXT, G_SEXT, G_ZEXT):$root,
-         [{ return Helper.matchCombineExtOfExt(*${root}, ${matchinfo}); }]),
-  (apply [{ Helper.applyCombineExtOfExt(*${root}, ${matchinfo}); }])
->;
-
 def not_cmp_fold_matchinfo : GIDefMatchData<"SmallVector<Register, 4>">;
 def not_cmp_fold : GICombineRule<
   (defs root:$d, not_cmp_fold_matchinfo:$info),
@@ -1850,13 +1841,36 @@ def select_of_zext : select_of_opcode<G_ZEXT>;
 def select_of_anyext : select_of_opcode<G_ANYEXT>;
 def select_of_truncate : select_of_opcode<G_TRUNC>;
 
+// Fold ([asz]ext ([asz]ext x)) -> ([asz]ext x).
+class ext_of_ext_opcodes<Instruction ext1Opcode, Instruction ext2Opcode> : GICombineRule <
+  (defs root:$root, build_fn_matchinfo:$matchinfo),
+  (match (ext2Opcode $second, $src):$Second,
+         (ext1Opcode $root, $second):$First,
+         [{ return Helper.matchExtOfExt(*${First}, *${Second}, ${matchinfo}); }]),
+  (apply [{ Helper.applyBuildFn(*${First}, ${matchinfo}); }])>;
+
+def zext_of_zext : ext_of_ext_opcodes<G_ZEXT, G_ZEXT>;
+def zext_of_anyext : ext_of_ext_opcodes<G_ZEXT, G_ANYEXT>;
+def sext_of_sext : ext_of_ext_opcodes<G_SEXT, G_SEXT>;
+def sext_of_anyext : ext_of_ext_opcodes<G_SEXT, G_ANYEXT>;
+def anyext_of_anyext : ext_of_ext_opcodes<G_ANYEXT, G_ANYEXT>;
+def anyext_of_zext : ext_of_ext_opcodes<G_ANYEXT, G_ZEXT>;
+def anyext_of_sext : ext_of_ext_opcodes<G_ANYEXT, G_SEXT>;
+
 def cast_combines: GICombineGroup<[
   truncate_of_zext,
   truncate_of_sext,
   truncate_of_anyext,
   select_of_zext,
   select_of_anyext,
-  select_of_truncate
+  select_of_truncate,
+  zext_of_zext,
+  zext_of_anyext,
+  sext_of_sext,
+  sext_of_anyext,
+  anyext_of_anyext,
+  anyext_of_zext,
+  anyext_of_sext
 ]>;
 
 
@@ -1928,7 +1942,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
     reassocs, ptr_add_immed_chain,
     shl_ashr_to_sext_inreg, sext_inreg_of_load,
     width_reduction_combines, select_combines,
-    known_bits_simplifications, ext_ext_fold,
+    known_bits_simplifications,
     not_cmp_fold, opt_brcond_by_inverting_cond,
     unmerge_merge, unmerge_cst, unmerge_dead_to_trunc,
     unmerge_zext_to_zext, merge_unmerge, trunc_shift,
diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h
index 885d3249fb4f833..1a4824a806dc6e0 100644
--- a/llvm/include/llvm/Transforms/Instrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation.h
@@ -46,7 +46,7 @@ BasicBlock::iterator PrepareToSplitEntryBlock(BasicBlock &BB,
 // Create a constant for Str so that we can pass it to the run-time lib.
 GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str,
                                              bool AllowMerging,
-                                             const char *NamePrefix = "");
+                                             Twine NamePrefix = "");
 
 // Returns F.getComdat() if it exists.
 // Otherwise creates a new comdat, sets F's comdat, and returns it.
diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
index 7199f27dbc991a8..1d0af87e965af5f 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
@@ -53,15 +53,18 @@ class PGOInstrumentationGenCreateVar
   bool ProfileSampling;
 };
 
+enum class PGOInstrumentationType { Invalid = 0, FDO, CSFDO, CTXPROF };
 /// The instrumentation (profile-instr-gen) pass for IR based PGO.
 class PGOInstrumentationGen : public PassInfoMixin<PGOInstrumentationGen> {
 public:
-  PGOInstrumentationGen(bool IsCS = false) : IsCS(IsCS) {}
+  PGOInstrumentationGen(
+      PGOInstrumentationType InstrumentationType = PGOInstrumentationType ::FDO)
+      : InstrumentationType(InstrumentationType) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
 private:
   // If this is a context sensitive instrumentation.
-  bool IsCS;
+  const PGOInstrumentationType InstrumentationType;
 };
 
 /// The profile annotation (profile-instr-use) pass for IR based PGO.
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index 98d0adca3552149..17f4327eb3e1aba 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -51,6 +51,14 @@ Pass *createLICMPass();
 //
 Pass *createLoopStrengthReducePass();
 
+//===----------------------------------------------------------------------===//
+//
+// LoopTermFold -  This pass attempts to eliminate the last use of an IV in
+// a loop terminator instruction by rewriting it in terms of another IV.
+// Expected to be run immediately after LSR.
+//
+Pass *createLoopTermFoldPass();
+
 //===----------------------------------------------------------------------===//
 //
 // LoopUnroll - This pass is a simple loop unrolling pass.
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopTermFold.h b/llvm/include/llvm/Transforms/Scalar/LoopTermFold.h
new file mode 100644
index 000000000000000..974024c586aa80b
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/LoopTermFold.h
@@ -0,0 +1,30 @@
+//===- LoopTermFold.h - Loop Term Fold Pass ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_SCALAR_LOOPTERMFOLD_H
+#define LLVM_TRANSFORMS_SCALAR_LOOPTERMFOLD_H
+
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Loop;
+class LPMUpdater;
+
+class LoopTermFoldPass : public PassInfoMixin<LoopTermFoldPass> {
+public:
+  PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
+                        LoopStandardAnalysisResults &AR, LPMUpdater &U);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_SCALAR_LOOPTERMFOLD_H
diff --git a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
index 429d6a2e05236fd..1979c4af770b02b 100644
--- a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
+++ b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
@@ -267,6 +267,13 @@ namespace llvm {
                                       IRBuilderBase &B,
                                       const TargetLibraryInfo *TLI,
                                       LibFunc NewFunc, uint8_t HotCold);
+  Value *emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
+                                     const TargetLibraryInfo *TLI,
+                                     LibFunc NewFunc, uint8_t HotCold);
+  Value *emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
+                                            IRBuilderBase &B,
+                                            const TargetLibraryInfo *TLI,
+                                            LibFunc NewFunc, uint8_t HotCold);
 }
 
 #endif
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 809beadb5f7df3c..95531544a1c8176 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -56,9 +56,9 @@ class BoUpSLP;
 } // end namespace slpvectorizer
 
 struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
-  using StoreList = SmallVector<StoreInst *, 4>;
+  using StoreList = SmallVector<StoreInst *, 8>;
   using StoreListMap = MapVector<Value *, StoreList>;
-  using GEPList = SmallVector<GetElementPtrInst *, 4>;
+  using GEPList = SmallVector<GetElementPtrInst *, 8>;
   using GEPListMap = MapVector<Value *, GEPList>;
   using InstSetVector = SmallSetVector<Instruction *, 8>;
 
diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp
index 7b4666b29a19369..d0ccf4ba537f847 100644
--- a/llvm/lib/Analysis/CtxProfAnalysis.cpp
+++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp
@@ -96,6 +96,9 @@ GlobalValue::GUID AssignGUIDPass::getGUID(const Function &F) {
 }
 AnalysisKey CtxProfAnalysis::Key;
 
+CtxProfAnalysis::CtxProfAnalysis(StringRef Profile)
+    : Profile(Profile.empty() ? UseCtxProfile : Profile) {}
+
 PGOContextualProfile CtxProfAnalysis::run(Module &M,
                                           ModuleAnalysisManager &MAM) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> MB = MemoryBuffer::getFile(Profile);
diff --git a/llvm/lib/Analysis/DXILResource.cpp b/llvm/lib/Analysis/DXILResource.cpp
index d7d10ec4d8d0d6c..1b5b051c9db29eb 100644
--- a/llvm/lib/Analysis/DXILResource.cpp
+++ b/llvm/lib/Analysis/DXILResource.cpp
@@ -23,7 +23,7 @@
 using namespace llvm;
 using namespace dxil;
 
-static constexpr StringRef getResourceClassName(ResourceClass RC) {
+static StringRef getResourceClassName(ResourceClass RC) {
   switch (RC) {
   case ResourceClass::SRV:
     return "SRV";
@@ -37,7 +37,7 @@ static constexpr StringRef getResourceClassName(ResourceClass RC) {
   llvm_unreachable("Unhandled ResourceClass");
 }
 
-static constexpr StringRef getResourceKindName(ResourceKind RK) {
+static StringRef getResourceKindName(ResourceKind RK) {
   switch (RK) {
   case ResourceKind::Texture1D:
     return "Texture1D";
@@ -82,7 +82,7 @@ static constexpr StringRef getResourceKindName(ResourceKind RK) {
   llvm_unreachable("Unhandled ResourceKind");
 }
 
-static constexpr StringRef getElementTypeName(ElementType ET) {
+static StringRef getElementTypeName(ElementType ET) {
   switch (ET) {
   case ElementType::I1:
     return "i1";
@@ -126,7 +126,7 @@ static constexpr StringRef getElementTypeName(ElementType ET) {
   llvm_unreachable("Unhandled ElementType");
 }
 
-static constexpr StringRef getSamplerTypeName(SamplerType ST) {
+static StringRef getSamplerTypeName(SamplerType ST) {
   switch (ST) {
   case SamplerType::Default:
     return "Default";
@@ -138,7 +138,7 @@ static constexpr StringRef getSamplerTypeName(SamplerType ST) {
   llvm_unreachable("Unhandled SamplerType");
 }
 
-static constexpr StringRef getSamplerFeedbackTypeName(SamplerFeedbackType SFT) {
+static StringRef getSamplerFeedbackTypeName(SamplerFeedbackType SFT) {
   switch (SFT) {
   case SamplerFeedbackType::MinMip:
     return "MinMip";
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index c4c174977c47589..32a9f1ab34fb3fc 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -30,6 +30,7 @@
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/OverflowInstAnalysis.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/ConstantRange.h"
@@ -3717,6 +3718,14 @@ static Value *simplifyICmpWithIntrinsicOnLHS(CmpInst::Predicate Pred,
       if (Pred == ICmpInst::ICMP_ULT)
         return ConstantInt::getFalse(getCompareTy(II));
     }
+    // uadd.sat(X, Y) uge X + Y
+    if (match(RHS, m_c_Add(m_Specific(II->getArgOperand(0)),
+                           m_Specific(II->getArgOperand(1))))) {
+      if (Pred == ICmpInst::ICMP_UGE)
+        return ConstantInt::getTrue(getCompareTy(II));
+      if (Pred == ICmpInst::ICMP_ULT)
+        return ConstantInt::getFalse(getCompareTy(II));
+    }
     return nullptr;
   case Intrinsic::usub_sat:
     // usub.sat(X, Y) ule X
@@ -3726,6 +3735,14 @@ static Value *simplifyICmpWithIntrinsicOnLHS(CmpInst::Predicate Pred,
       if (Pred == ICmpInst::ICMP_UGT)
         return ConstantInt::getFalse(getCompareTy(II));
     }
+    // usub.sat(X, Y) ule X - Y
+    if (match(RHS, m_Sub(m_Specific(II->getArgOperand(0)),
+                         m_Specific(II->getArgOperand(1))))) {
+      if (Pred == ICmpInst::ICMP_ULE)
+        return ConstantInt::getTrue(getCompareTy(II));
+      if (Pred == ICmpInst::ICMP_UGT)
+        return ConstantInt::getFalse(getCompareTy(II));
+    }
     return nullptr;
   default:
     return nullptr;
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index d67fd7985f6a505..872bc52b82cca7d 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -412,12 +412,10 @@ bool RuntimePointerChecking::needsChecking(
 /// Return nullptr in case we couldn't find an answer.
 static const SCEV *getMinFromExprs(const SCEV *I, const SCEV *J,
                                    ScalarEvolution *SE) {
-  const SCEV *Diff = SE->getMinusSCEV(J, I);
-  const SCEVConstant *C = dyn_cast<const SCEVConstant>(Diff);
-
-  if (!C)
+  std::optional<APInt> Diff = SE->computeConstantDifference(J, I);
+  if (!Diff)
     return nullptr;
-  return C->getValue()->isNegative() ? J : I;
+  return Diff->isNegative() ? J : I;
 }
 
 bool RuntimeCheckingPtrGroup::addPointer(
@@ -1599,11 +1597,11 @@ std::optional<int> llvm::getPointersDiff(Type *ElemTyA, Value *PtrA,
     // Otherwise compute the distance with SCEV between the base pointers.
     const SCEV *PtrSCEVA = SE.getSCEV(PtrA);
     const SCEV *PtrSCEVB = SE.getSCEV(PtrB);
-    const auto *Diff =
-        dyn_cast<SCEVConstant>(SE.getMinusSCEV(PtrSCEVB, PtrSCEVA));
+    std::optional<APInt> Diff =
+        SE.computeConstantDifference(PtrSCEVB, PtrSCEVA);
     if (!Diff)
       return std::nullopt;
-    Val = Diff->getAPInt().getSExtValue();
+    Val = Diff->getSExtValue();
   }
   int Size = DL.getTypeStoreSize(ElemTyA);
   int Dist = Val / Size;
diff --git a/llvm/lib/Analysis/MemoryProfileInfo.cpp b/llvm/lib/Analysis/MemoryProfileInfo.cpp
index c81c65fd4e691e3..3a752d42241a95e 100644
--- a/llvm/lib/Analysis/MemoryProfileInfo.cpp
+++ b/llvm/lib/Analysis/MemoryProfileInfo.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/MemoryProfileInfo.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 487844f000ac695..a19358dee8ef491 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -11956,7 +11956,7 @@ ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
   APInt DiffMul(BW, 1);
   // Try various simplifications to reduce the difference to a constant. Limit
   // the number of allowed simplifications to keep compile-time low.
-  for (unsigned I = 0; I < 5; ++I) {
+  for (unsigned I = 0; I < 8; ++I) {
     if (More == Less)
       return Diff;
 
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index dcde78925bfa984..2c26493bd3f1ca2 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -427,10 +427,6 @@ bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const {
   return TTIImpl->isNumRegsMajorCostOfLSR();
 }
 
-bool TargetTransformInfo::shouldFoldTerminatingConditionAfterLSR() const {
-  return TTIImpl->shouldFoldTerminatingConditionAfterLSR();
-}
-
 bool TargetTransformInfo::shouldDropLSRSolutionIfLessProfitable() const {
   return TTIImpl->shouldDropLSRSolutionIfLessProfitable();
 }
diff --git a/llvm/lib/Analysis/ValueLattice.cpp b/llvm/lib/Analysis/ValueLattice.cpp
index 1d2177a92eb4654..5001fc2d310dd43 100644
--- a/llvm/lib/Analysis/ValueLattice.cpp
+++ b/llvm/lib/Analysis/ValueLattice.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Analysis/ValueLattice.h"
 #include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/IR/Instructions.h"
 
 namespace llvm {
 Constant *
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index b6414a3e7990b0a..173faa32a3878d8 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -310,8 +310,9 @@ bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
                            const DataLayout &DL, AssumptionCache *AC,
                            const Instruction *CxtI, const DominatorTree *DT,
                            bool UseInstrInfo) {
-  assert(V1->getType() == V2->getType() &&
-         "Testing equality of non-equal types!");
+  // We don't support looking through casts.
+  if (V1 == V2 || V1->getType() != V2->getType())
+    return false;
   auto *FVTy = dyn_cast<FixedVectorType>(V1->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
@@ -6035,8 +6036,8 @@ Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
 
   if (isa<ConstantAggregate>(C)) {
     Value *Val = UndefInt8;
-    for (unsigned I = 0, E = C->getNumOperands(); I != E; ++I)
-      if (!(Val = Merge(Val, isBytewiseValue(C->getOperand(I), DL))))
+    for (Value *Op : C->operands())
+      if (!(Val = Merge(Val, isBytewiseValue(Op, DL))))
         return nullptr;
     return Val;
   }
@@ -9671,7 +9672,7 @@ static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
   if (!I->getOperand(0)->getType()->getScalarType()->isHalfTy())
     return;
   if (isa<FPToSIInst>(I) && BitWidth >= 17) {
-    Lower = APInt(BitWidth, -65504);
+    Lower = APInt(BitWidth, -65504, true);
     Upper = APInt(BitWidth, 65505);
   }
 
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 460db6fc01de4a0..03d0537291dada1 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -2820,9 +2820,9 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
           Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
           emitConstantRange(Record, *Range, /*EmitBitWidth=*/true);
         }
-        for (unsigned i = 0, e = CE->getNumOperands(); i != e; ++i) {
-          Record.push_back(VE.getTypeID(C->getOperand(i)->getType()));
-          Record.push_back(VE.getValueID(C->getOperand(i)));
+        for (const Value *Op : CE->operands()) {
+          Record.push_back(VE.getTypeID(Op->getType()));
+          Record.push_back(VE.getValueID(Op));
         }
         break;
       }
@@ -3078,8 +3078,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
         if (!pushValueAndType(I.getOperand(0), InstID, Vals))
           AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
       } else {
-        for (unsigned i = 0, e = NumOperands; i != e; ++i)
-          pushValueAndType(I.getOperand(i), InstID, Vals);
+        for (const Value *Op : I.operands())
+          pushValueAndType(Op, InstID, Vals);
       }
     }
     break;
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 7ab2f2c13def879..60cb26973ead414 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -636,7 +636,7 @@ void AsmPrinter::emitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const {
   case GlobalValue::LinkOnceODRLinkage:
   case GlobalValue::WeakAnyLinkage:
   case GlobalValue::WeakODRLinkage:
-    if (MAI->hasWeakDefDirective()) {
+    if (MAI->isMachO()) {
       // .globl _foo
       OutStreamer->emitSymbolAttribute(GVSym, MCSA_Global);
 
@@ -779,8 +779,7 @@ void AsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
 
   // If we have a bss global going to a section that supports the
   // zerofill directive, do so here.
-  if (GVKind.isBSS() && MAI->hasMachoZeroFillDirective() &&
-      TheSection->isVirtualSection()) {
+  if (GVKind.isBSS() && MAI->isMachO() && TheSection->isVirtualSection()) {
     if (Size == 0)
       Size = 1; // zerofill of 0 bytes is undefined.
     emitLinkage(GV, GVSym);
@@ -825,7 +824,7 @@ void AsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
   // TLOF class.  This will also make it more obvious that stuff like
   // MCStreamer::EmitTBSSSymbol is macho specific and only called from macho
   // specific code.
-  if (GVKind.isThreadLocal() && MAI->hasMachoTBSSDirective()) {
+  if (GVKind.isThreadLocal() && MAI->isMachO()) {
     // Emit the .tbss symbol
     MCSymbol *MangSym =
         OutContext.getOrCreateSymbol(GVSym->getName() + Twine("$tlv$init"));
@@ -2148,7 +2147,7 @@ void AsmPrinter::emitGlobalAlias(const Module &M, const GlobalAlias &GA) {
 
   const MCExpr *Expr = lowerConstant(GA.getAliasee());
 
-  if (MAI->hasAltEntry() && isa<MCBinaryExpr>(Expr))
+  if (MAI->isMachO() && isa<MCBinaryExpr>(Expr))
     OutStreamer->emitSymbolAttribute(Name, MCSA_AltEntry);
 
   // Emit the directives as assignments aka .set:
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 1fb37fb8406ef45..48253a613b41d2b 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -7382,12 +7383,9 @@ bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) {
     if (IsHugeFunc) {
       // Now we clone an instruction, its operands' defs may sink to this BB
       // now. So we put the operands defs' BBs into FreshBBs to do optimization.
-      for (unsigned I = 0; I < NI->getNumOperands(); ++I) {
-        auto *OpDef = dyn_cast<Instruction>(NI->getOperand(I));
-        if (!OpDef)
-          continue;
-        FreshBBs.insert(OpDef->getParent());
-      }
+      for (Value *Op : NI->operands())
+        if (auto *OpDef = dyn_cast<Instruction>(Op))
+          FreshBBs.insert(OpDef->getParent());
     }
 
     NewInstructions[UI] = NI;
diff --git a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
index 49842e5fd65da6f..c5ec73cd5c65d83 100644
--- a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
@@ -45,61 +45,189 @@ cl::OptionCategory GICombinerOptionCategory(
 );
 } // end namespace llvm
 
-/// This class acts as the glue the joins the CombinerHelper to the overall
+/// This class acts as the glue that joins the CombinerHelper to the overall
 /// Combine algorithm. The CombinerHelper is intended to report the
 /// modifications it makes to the MIR to the GISelChangeObserver and the
-/// observer subclass will act on these events. In this case, instruction
-/// erasure will cancel any future visits to the erased instruction and
-/// instruction creation will schedule that instruction for a future visit.
-/// Other Combiner implementations may require more complex behaviour from
-/// their GISelChangeObserver subclass.
+/// observer subclass will act on these events.
 class Combiner::WorkListMaintainer : public GISelChangeObserver {
-  using WorkListTy = GISelWorkList<512>;
-  WorkListTy &WorkList;
+protected:
+#ifndef NDEBUG
   /// The instructions that have been created but we want to report once they
   /// have their operands. This is only maintained if debug output is requested.
-#ifndef NDEBUG
-  SetVector<const MachineInstr *> CreatedInstrs;
+  SmallSetVector<const MachineInstr *, 32> CreatedInstrs;
 #endif
+  using Level = CombinerInfo::ObserverLevel;
 
 public:
-  WorkListMaintainer(WorkListTy &WorkList) : WorkList(WorkList) {}
+  static std::unique_ptr<WorkListMaintainer>
+  create(Level Lvl, WorkListTy &WorkList, MachineRegisterInfo &MRI);
+
   virtual ~WorkListMaintainer() = default;
 
+  void reportFullyCreatedInstrs() {
+    LLVM_DEBUG({
+      for (auto *MI : CreatedInstrs) {
+        dbgs() << "Created: " << *MI;
+      }
+      CreatedInstrs.clear();
+    });
+  }
+
+  virtual void reset() = 0;
+  virtual void appliedCombine() = 0;
+};
+
+/// A configurable WorkListMaintainer implementation.
+/// The ObserverLevel determines how the WorkListMaintainer reacts to MIR
+/// changes.
+template <CombinerInfo::ObserverLevel Lvl>
+class Combiner::WorkListMaintainerImpl : public Combiner::WorkListMaintainer {
+  WorkListTy &WorkList;
+  MachineRegisterInfo &MRI;
+
+  // Defer handling these instructions until the combine finishes.
+  SmallSetVector<MachineInstr *, 32> DeferList;
+
+  // Track VRegs that (might) have lost a use.
+  SmallSetVector<Register, 32> LostUses;
+
+public:
+  WorkListMaintainerImpl(WorkListTy &WorkList, MachineRegisterInfo &MRI)
+      : WorkList(WorkList), MRI(MRI) {}
+
+  virtual ~WorkListMaintainerImpl() = default;
+
+  void reset() override {
+    DeferList.clear();
+    LostUses.clear();
+  }
+
   void erasingInstr(MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Erasing: " << MI << "\n");
+    // MI will become dangling, remove it from all lists.
+    LLVM_DEBUG(dbgs() << "Erasing: " << MI; CreatedInstrs.remove(&MI));
     WorkList.remove(&MI);
+    if constexpr (Lvl != Level::Basic) {
+      DeferList.remove(&MI);
+      noteLostUses(MI);
+    }
   }
+
   void createdInstr(MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Creating: " << MI << "\n");
-    WorkList.insert(&MI);
-    LLVM_DEBUG(CreatedInstrs.insert(&MI));
+    LLVM_DEBUG(dbgs() << "Creating: " << MI; CreatedInstrs.insert(&MI));
+    if constexpr (Lvl == Level::Basic)
+      WorkList.insert(&MI);
+    else
+      // Defer handling newly created instructions, because they don't have
+      // operands yet. We also insert them into the WorkList in reverse
+      // order so that they will be combined top down.
+      DeferList.insert(&MI);
   }
+
   void changingInstr(MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Changing: " << MI << "\n");
-    WorkList.insert(&MI);
+    LLVM_DEBUG(dbgs() << "Changing: " << MI);
+    // Some uses might get dropped when MI is changed.
+    // For now, overapproximate by assuming all uses will be dropped.
+    // TODO: Is a more precise heuristic or manual tracking of use count
+    // decrements worth it?
+    if constexpr (Lvl != Level::Basic)
+      noteLostUses(MI);
   }
+
   void changedInstr(MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Changed: " << MI << "\n");
-    WorkList.insert(&MI);
+    LLVM_DEBUG(dbgs() << "Changed: " << MI);
+    if constexpr (Lvl == Level::Basic)
+      WorkList.insert(&MI);
+    else
+      // Defer this for DCE
+      DeferList.insert(&MI);
   }
 
-  void reportFullyCreatedInstrs() {
-    LLVM_DEBUG(for (const auto *MI
-                    : CreatedInstrs) {
-      dbgs() << "Created: ";
-      MI->print(dbgs());
-    });
-    LLVM_DEBUG(CreatedInstrs.clear());
+  // Only track changes during the combine and then walk the def/use-chains once
+  // the combine is finished, because:
+  // - instructions might have multiple defs during the combine.
+  // - use counts aren't accurate during the combine.
+  void appliedCombine() override {
+    if constexpr (Lvl == Level::Basic)
+      return;
+
+    // DCE deferred instructions and add them to the WorkList bottom up.
+    while (!DeferList.empty()) {
+      MachineInstr &MI = *DeferList.pop_back_val();
+      if (tryDCE(MI, MRI))
+        continue;
+
+      if constexpr (Lvl >= Level::SinglePass)
+        addUsersToWorkList(MI);
+
+      WorkList.insert(&MI);
+    }
+
+    // Handle instructions that have lost a user.
+    while (!LostUses.empty()) {
+      Register Use = LostUses.pop_back_val();
+      MachineInstr *UseMI = MRI.getVRegDef(Use);
+      if (!UseMI)
+        continue;
+
+      // If DCE succeeds, UseMI's uses are added back to LostUses by
+      // erasingInstr.
+      if (tryDCE(*UseMI, MRI))
+        continue;
+
+      if constexpr (Lvl >= Level::SinglePass) {
+        // OneUse checks are relatively common, so we might be able to combine
+        // the single remaining user of this Reg.
+        if (MRI.hasOneNonDBGUser(Use))
+          WorkList.insert(&*MRI.use_instr_nodbg_begin(Use));
+
+        WorkList.insert(UseMI);
+      }
+    }
+  }
+
+  void noteLostUses(MachineInstr &MI) {
+    for (auto &Use : MI.explicit_uses()) {
+      if (!Use.isReg() || !Use.getReg().isVirtual())
+        continue;
+      LostUses.insert(Use.getReg());
+    }
+  }
+
+  void addUsersToWorkList(MachineInstr &MI) {
+    for (auto &Def : MI.defs()) {
+      Register DefReg = Def.getReg();
+      if (!DefReg.isVirtual())
+        continue;
+      for (auto &UseMI : MRI.use_nodbg_instructions(DefReg)) {
+        WorkList.insert(&UseMI);
+      }
+    }
   }
 };
 
+std::unique_ptr<Combiner::WorkListMaintainer>
+Combiner::WorkListMaintainer::create(Level Lvl, WorkListTy &WorkList,
+                                     MachineRegisterInfo &MRI) {
+  switch (Lvl) {
+  case Level::Basic:
+    return std::make_unique<WorkListMaintainerImpl<Level::Basic>>(WorkList,
+                                                                  MRI);
+  case Level::DCE:
+    return std::make_unique<WorkListMaintainerImpl<Level::DCE>>(WorkList, MRI);
+  case Level::SinglePass:
+    return std::make_unique<WorkListMaintainerImpl<Level::SinglePass>>(WorkList,
+                                                                       MRI);
+  }
+  llvm_unreachable("Illegal ObserverLevel");
+}
+
 Combiner::Combiner(MachineFunction &MF, CombinerInfo &CInfo,
                    const TargetPassConfig *TPC, GISelKnownBits *KB,
                    GISelCSEInfo *CSEInfo)
     : Builder(CSEInfo ? std::make_unique<CSEMIRBuilder>()
                       : std::make_unique<MachineIRBuilder>()),
-      WLObserver(std::make_unique<WorkListMaintainer>(WorkList)),
+      WLObserver(WorkListMaintainer::create(CInfo.ObserverLvl, WorkList,
+                                            MF.getRegInfo())),
       ObserverWrapper(std::make_unique<GISelObserverWrapper>()), CInfo(CInfo),
       Observer(*ObserverWrapper), B(*Builder), MF(MF), MRI(MF.getRegInfo()),
       KB(KB), TPC(TPC), CSEInfo(CSEInfo) {
@@ -115,6 +243,15 @@ Combiner::Combiner(MachineFunction &MF, CombinerInfo &CInfo,
 
 Combiner::~Combiner() = default;
 
+bool Combiner::tryDCE(MachineInstr &MI, MachineRegisterInfo &MRI) {
+  if (!isTriviallyDead(MI, MRI))
+    return false;
+  LLVM_DEBUG(dbgs() << "Dead: " << MI);
+  llvm::salvageDebugInfo(MRI, MI);
+  MI.eraseFromParent();
+  return true;
+}
+
 bool Combiner::combineMachineInstrs() {
   // If the ISel pipeline failed, do not bother running this pass.
   // FIXME: Should this be here or in individual combiner passes.
@@ -141,27 +278,29 @@ bool Combiner::combineMachineInstrs() {
     ++Iteration;
     LLVM_DEBUG(dbgs() << "\n\nCombiner iteration #" << Iteration << '\n');
 
+    Changed = false;
     WorkList.clear();
+    WLObserver->reset();
     ObserverWrapper->clearObservers();
     if (CSEInfo)
       ObserverWrapper->addObserver(CSEInfo);
 
+    // If Observer-based DCE is enabled, perform full DCE only before the first
+    // iteration.
+    bool EnableDCE = CInfo.ObserverLvl >= CombinerInfo::ObserverLevel::DCE
+                         ? CInfo.EnableFullDCE && Iteration == 1
+                         : CInfo.EnableFullDCE;
+
     // Collect all instructions. Do a post order traversal for basic blocks and
     // insert with list bottom up, so while we pop_back_val, we'll traverse top
     // down RPOT.
-    Changed = false;
-
     RAIIMFObsDelInstaller DelInstall(MF, *ObserverWrapper);
     for (MachineBasicBlock *MBB : post_order(&MF)) {
       for (MachineInstr &CurMI :
            llvm::make_early_inc_range(llvm::reverse(*MBB))) {
         // Erase dead insts before even adding to the list.
-        if (isTriviallyDead(CurMI, MRI)) {
-          LLVM_DEBUG(dbgs() << CurMI << "Is dead; erasing.\n");
-          llvm::salvageDebugInfo(MRI, CurMI);
-          CurMI.eraseFromParent();
+        if (EnableDCE && tryDCE(CurMI, MRI))
           continue;
-        }
         WorkList.deferred_insert(&CurMI);
       }
     }
@@ -171,10 +310,13 @@ bool Combiner::combineMachineInstrs() {
     ObserverWrapper->addObserver(WLObserver.get());
     // Main Loop. Process the instructions here.
     while (!WorkList.empty()) {
-      MachineInstr *CurrInst = WorkList.pop_back_val();
-      LLVM_DEBUG(dbgs() << "\nTry combining " << *CurrInst;);
-      Changed |= tryCombineAll(*CurrInst);
-      WLObserver->reportFullyCreatedInstrs();
+      MachineInstr &CurrInst = *WorkList.pop_back_val();
+      LLVM_DEBUG(dbgs() << "\nTry combining " << CurrInst);
+      bool AppliedCombine = tryCombineAll(CurrInst);
+      LLVM_DEBUG(WLObserver->reportFullyCreatedInstrs());
+      Changed |= AppliedCombine;
+      if (AppliedCombine)
+        WLObserver->appliedCombine();
     }
     MFChanged |= Changed;
 
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index d363e6794cb017b..1517ae707c8cff1 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -2538,60 +2538,6 @@ bool CombinerHelper::matchCombineZextTrunc(MachineInstr &MI, Register &Reg) {
   return false;
 }
 
-bool CombinerHelper::matchCombineExtOfExt(
-    MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) {
-  assert((MI.getOpcode() == TargetOpcode::G_ANYEXT ||
-          MI.getOpcode() == TargetOpcode::G_SEXT ||
-          MI.getOpcode() == TargetOpcode::G_ZEXT) &&
-         "Expected a G_[ASZ]EXT");
-  Register SrcReg = MI.getOperand(1).getReg();
-  Register OriginalSrcReg = getSrcRegIgnoringCopies(SrcReg, MRI);
-  if (OriginalSrcReg.isValid())
-    SrcReg = OriginalSrcReg;
-  MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
-  // Match exts with the same opcode, anyext([sz]ext) and sext(zext).
-  unsigned Opc = MI.getOpcode();
-  unsigned SrcOpc = SrcMI->getOpcode();
-  if (Opc == SrcOpc ||
-      (Opc == TargetOpcode::G_ANYEXT &&
-       (SrcOpc == TargetOpcode::G_SEXT || SrcOpc == TargetOpcode::G_ZEXT)) ||
-      (Opc == TargetOpcode::G_SEXT && SrcOpc == TargetOpcode::G_ZEXT)) {
-    MatchInfo = std::make_tuple(SrcMI->getOperand(1).getReg(), SrcOpc);
-    return true;
-  }
-  return false;
-}
-
-void CombinerHelper::applyCombineExtOfExt(
-    MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) {
-  assert((MI.getOpcode() == TargetOpcode::G_ANYEXT ||
-          MI.getOpcode() == TargetOpcode::G_SEXT ||
-          MI.getOpcode() == TargetOpcode::G_ZEXT) &&
-         "Expected a G_[ASZ]EXT");
-
-  Register Reg = std::get<0>(MatchInfo);
-  unsigned SrcExtOp = std::get<1>(MatchInfo);
-
-  // Combine exts with the same opcode.
-  if (MI.getOpcode() == SrcExtOp) {
-    Observer.changingInstr(MI);
-    MI.getOperand(1).setReg(Reg);
-    Observer.changedInstr(MI);
-    return;
-  }
-
-  // Combine:
-  // - anyext([sz]ext x) to [sz]ext x
-  // - sext(zext x) to zext x
-  if (MI.getOpcode() == TargetOpcode::G_ANYEXT ||
-      (MI.getOpcode() == TargetOpcode::G_SEXT &&
-       SrcExtOp == TargetOpcode::G_ZEXT)) {
-    Register DstReg = MI.getOperand(0).getReg();
-    Builder.buildInstr(SrcExtOp, {DstReg}, {Reg});
-    MI.eraseFromParent();
-  }
-}
-
 static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy) {
   const unsigned ShiftSize = ShiftTy.getScalarSizeInBits();
   const unsigned TruncSize = TruncTy.getScalarSizeInBits();
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
index 59295f7a65835c6..494d8da84445d15 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
@@ -209,3 +209,67 @@ bool CombinerHelper::matchCastOfSelect(const MachineInstr &CastMI,
 
   return true;
 }
+
+bool CombinerHelper::matchExtOfExt(const MachineInstr &FirstMI,
+                                   const MachineInstr &SecondMI,
+                                   BuildFnTy &MatchInfo) {
+  const GExtOp *First = cast<GExtOp>(&FirstMI);
+  const GExtOp *Second = cast<GExtOp>(&SecondMI);
+
+  Register Dst = First->getReg(0);
+  Register Src = Second->getSrcReg();
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+
+  if (!MRI.hasOneNonDBGUse(Second->getReg(0)))
+    return false;
+
+  // ext of ext -> later ext
+  if (First->getOpcode() == Second->getOpcode() &&
+      isLegalOrBeforeLegalizer({Second->getOpcode(), {DstTy, SrcTy}})) {
+    if (Second->getOpcode() == TargetOpcode::G_ZEXT) {
+      MachineInstr::MIFlag Flag = MachineInstr::MIFlag::NoFlags;
+      if (Second->getFlag(MachineInstr::MIFlag::NonNeg))
+        Flag = MachineInstr::MIFlag::NonNeg;
+      MatchInfo = [=](MachineIRBuilder &B) { B.buildZExt(Dst, Src, Flag); };
+      return true;
+    }
+    // not zext -> no flags
+    MatchInfo = [=](MachineIRBuilder &B) {
+      B.buildInstr(Second->getOpcode(), {Dst}, {Src});
+    };
+    return true;
+  }
+
+  // anyext of sext/zext  -> sext/zext
+  // -> pick anyext as second ext, then ext of ext
+  if (First->getOpcode() == TargetOpcode::G_ANYEXT &&
+      isLegalOrBeforeLegalizer({Second->getOpcode(), {DstTy, SrcTy}})) {
+    if (Second->getOpcode() == TargetOpcode::G_ZEXT) {
+      MachineInstr::MIFlag Flag = MachineInstr::MIFlag::NoFlags;
+      if (Second->getFlag(MachineInstr::MIFlag::NonNeg))
+        Flag = MachineInstr::MIFlag::NonNeg;
+      MatchInfo = [=](MachineIRBuilder &B) { B.buildZExt(Dst, Src, Flag); };
+      return true;
+    }
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildSExt(Dst, Src); };
+    return true;
+  }
+
+  // sext/zext of anyext -> sext/zext
+  // -> pick anyext as first ext, then ext of ext
+  if (Second->getOpcode() == TargetOpcode::G_ANYEXT &&
+      isLegalOrBeforeLegalizer({First->getOpcode(), {DstTy, SrcTy}})) {
+    if (First->getOpcode() == TargetOpcode::G_ZEXT) {
+      MachineInstr::MIFlag Flag = MachineInstr::MIFlag::NoFlags;
+      if (First->getFlag(MachineInstr::MIFlag::NonNeg))
+        Flag = MachineInstr::MIFlag::NonNeg;
+      MatchInfo = [=](MachineIRBuilder &B) { B.buildZExt(Dst, Src, Flag); };
+      return true;
+    }
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildSExt(Dst, Src); };
+    return true;
+  }
+
+  return false;
+}
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 219c60eab04f5ac..f44af78cded46de 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1463,8 +1463,8 @@ static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
     for (auto Idx : IVI->indices())
       Indices.push_back(ConstantInt::get(Int32Ty, Idx));
   } else {
-    for (unsigned i = 1; i < U.getNumOperands(); ++i)
-      Indices.push_back(U.getOperand(i));
+    for (Value *Op : drop_begin(U.operands()))
+      Indices.push_back(Op);
   }
 
   return 8 * static_cast<uint64_t>(
@@ -3889,6 +3889,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to translate in big endian mode";
     reportTranslationError(*MF, *TPC, *ORE, R);
+    return false;
   }
 
   // Release the per-function state when we return, whether we succeeded or not.
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 225ec19246231da..bdbef20e20960d3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2523,7 +2523,8 @@ LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
 
 LegalizerHelper::LegalizeResult
 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
-  switch (MI.getOpcode()) {
+  unsigned Opcode = MI.getOpcode();
+  switch (Opcode) {
   default:
     return UnableToLegalize;
   case TargetOpcode::G_ATOMICRMW_XCHG:
@@ -2607,13 +2608,13 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Register SrcReg = MI.getOperand(1).getReg();
 
     // First extend the input.
-    unsigned ExtOpc = MI.getOpcode() == TargetOpcode::G_CTTZ ||
-                              MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF
+    unsigned ExtOpc = Opcode == TargetOpcode::G_CTTZ ||
+                              Opcode == TargetOpcode::G_CTTZ_ZERO_UNDEF
                           ? TargetOpcode::G_ANYEXT
                           : TargetOpcode::G_ZEXT;
     auto MIBSrc = MIRBuilder.buildInstr(ExtOpc, {WideTy}, {SrcReg});
     LLT CurTy = MRI.getType(SrcReg);
-    unsigned NewOpc = MI.getOpcode();
+    unsigned NewOpc = Opcode;
     if (NewOpc == TargetOpcode::G_CTTZ) {
       // The count is the same in the larger type except if the original
       // value was zero.  This can be handled by setting the bit just off
@@ -2628,7 +2629,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
 
     unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
 
-    if (MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
+    if (Opcode == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
       // An optimization where the result is the CTLZ after the left shift by
       // (Difference in widety and current ty), that is,
       // MIBSrc = MIBSrc << (sizeinbits(WideTy) - sizeinbits(CurTy))
@@ -2640,7 +2641,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     // Perform the operation at the larger size.
     auto MIBNewOp = MIRBuilder.buildInstr(NewOpc, {WideTy}, {MIBSrc});
     // This is already the correct result for CTPOP and CTTZs
-    if (MI.getOpcode() == TargetOpcode::G_CTLZ) {
+    if (Opcode == TargetOpcode::G_CTLZ) {
       // The correct result is NewOp - (Difference in widety and current ty).
       MIBNewOp = MIRBuilder.buildSub(
           WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff));
@@ -2788,8 +2789,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changingInstr(MI);
 
     if (TypeIdx == 0) {
-      unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
-        TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
+      unsigned CvtOp = Opcode == TargetOpcode::G_ASHR ? TargetOpcode::G_SEXT
+                                                      : TargetOpcode::G_ZEXT;
 
       widenScalarSrc(MI, WideTy, 1, CvtOp);
       widenScalarDst(MI, WideTy);
@@ -3101,7 +3102,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_FLDEXP:
   case TargetOpcode::G_STRICT_FLDEXP: {
     if (TypeIdx == 0) {
-      if (MI.getOpcode() == TargetOpcode::G_STRICT_FLDEXP)
+      if (Opcode == TargetOpcode::G_STRICT_FLDEXP)
         return UnableToLegalize;
 
       Observer.changingInstr(MI);
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index dae0cb2c900e5be..78201d9bfb79a97 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -130,6 +130,7 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
   // Generate the prolog instructions that set up the pipeline.
   generateProlog(MaxStageCount, KernelBB, VRMap, PrologBBs);
   MF.insert(BB->getIterator(), KernelBB);
+  LIS.insertMBBInMaps(KernelBB);
 
   // Rearrange the instructions to generate the new, pipelined loop,
   // and update register names as needed.
@@ -210,6 +211,7 @@ void ModuloScheduleExpander::generateProlog(unsigned LastStage,
     NewBB->transferSuccessors(PredBB);
     PredBB->addSuccessor(NewBB);
     PredBB = NewBB;
+    LIS.insertMBBInMaps(NewBB);
 
     // Generate instructions for each appropriate stage. Process instructions
     // in original program order.
@@ -283,6 +285,7 @@ void ModuloScheduleExpander::generateEpilog(
 
     PredBB->replaceSuccessor(LoopExitBB, NewBB);
     NewBB->addSuccessor(LoopExitBB);
+    LIS.insertMBBInMaps(NewBB);
 
     if (EpilogStart == LoopExitBB)
       EpilogStart = NewBB;
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index ee03eaa8ae527c6..c03ea5878052130 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -1553,7 +1553,7 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
       // If this instruction has a FrameIndex operand, we need to
       // use that target machine register info object to eliminate
       // it.
-      TRI.eliminateFrameIndex(MI, SPAdj, i);
+      TRI.eliminateFrameIndex(MI, SPAdj, i, RS);
 
       // Reset the iterator if we were at the beginning of the BB.
       if (AtBeginning) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 25644c24855a62f..80fdedcf9c62599 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -9561,7 +9561,7 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
   // A rotate left of ~1 is a nice way of achieving the desired result.
   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL &&
       isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
-    return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
+    return DAG.getNode(ISD::ROTL, DL, VT, DAG.getSignedConstant(~1, DL, VT),
                        N0.getOperand(1));
   }
 
@@ -13201,9 +13201,7 @@ SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
   unsigned CastOpcode = Cast->getOpcode();
   assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||
           CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
-          CastOpcode == ISD::TRUNCATE_SSAT_S ||
-          CastOpcode == ISD::TRUNCATE_SSAT_U ||
-          CastOpcode == ISD::TRUNCATE_USAT_U || CastOpcode == ISD::FP_ROUND) &&
+          CastOpcode == ISD::FP_ROUND) &&
          "Unexpected opcode for vector select narrowing/widening");
 
   // We only do this transform before legal ops because the pattern may be
@@ -18301,7 +18299,7 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
   // nondeterministic jumps).
   if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse()) {
     return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, Chain,
-                       N1->getOperand(0), N2);
+                       N1->getOperand(0), N2, N->getFlags());
   }
 
   // Variant of the previous fold where there is a SETCC in between:
@@ -18350,7 +18348,8 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
     if (Updated)
       return DAG.getNode(
           ISD::BRCOND, SDLoc(N), MVT::Other, Chain,
-          DAG.getSetCC(SDLoc(N1), N1->getValueType(0), S0, S1, Cond), N2);
+          DAG.getSetCC(SDLoc(N1), N1->getValueType(0), S0, S1, Cond), N2,
+          N->getFlags());
   }
 
   // If N is a constant we could fold this into a fallthrough or unconditional
@@ -18375,7 +18374,7 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
     HandleSDNode ChainHandle(Chain);
     if (SDValue NewN1 = rebuildSetCC(N1))
       return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other,
-                         ChainHandle.getValue(), NewN1, N2);
+                         ChainHandle.getValue(), NewN1, N2, N->getFlags());
   }
 
   return SDValue();
@@ -27140,6 +27139,10 @@ static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG,
                       : TLI.getTypeToTransformTo(*DAG.getContext(), EltVT)))
     return SDValue();
 
+  // FIXME: Type legalization can't handle illegal MULHS/MULHU.
+  if ((Opcode == ISD::MULHS || Opcode == ISD::MULHU) && !TLI.isTypeLegal(EltVT))
+    return SDValue();
+
   SDValue IndexC = DAG.getVectorIdxConstant(Index0, DL);
   SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src0, IndexC);
   SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src1, IndexC);
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 4ce92e156cf85ea..db33d5242601e07 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -1065,14 +1065,17 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
   // Create the new machine instruction.
   MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II);
 
+  // Transfer IR flags from the SDNode to the MachineInstr
+  MachineInstr *MI = MIB.getInstr();
+  const SDNodeFlags Flags = Node->getFlags();
+  if (Flags.hasUnpredictable())
+    MI->setFlag(MachineInstr::MIFlag::Unpredictable);
+
   // Add result register values for things that are defined by this
   // instruction.
   if (NumResults) {
     CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
 
-    // Transfer any IR flags from the SDNode to the MachineInstr
-    MachineInstr *MI = MIB.getInstr();
-    const SDNodeFlags Flags = Node->getFlags();
     if (Flags.hasNoSignedZeros())
       MI->setFlag(MachineInstr::MIFlag::FmNsz);
 
@@ -1105,9 +1108,6 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
 
     if (Flags.hasNoFPExcept())
       MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
-
-    if (Flags.hasUnpredictable())
-      MI->setFlag(MachineInstr::MIFlag::Unpredictable);
   }
 
   // Emit all of the actual operands of this instruction, adding them to the
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index e7f765382b0e469..74e3a898569beaf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -2601,7 +2601,7 @@ SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
   SDValue IsDenormal =
       DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
 
-  SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
+  SDValue MinExp = DAG.getSignedConstant(MinExpVal, dl, ExpVT);
   SDValue Zero = DAG.getConstant(0, dl, ExpVT);
 
   SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
@@ -5042,7 +5042,8 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
   case ISD::RESET_FPENV: {
     // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
     // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
-    SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
+    EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
+    SDValue Ptr = DAG.getAllOnesConstant(dl, PtrTy);
     SDValue Chain = Node->getOperand(0);
     Results.push_back(
         DAG.makeStateFunctionCall(RTLIB::FESETENV, Ptr, Chain, dl));
@@ -5097,7 +5098,7 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
     // target must provide custom lowering.
     const DataLayout &DL = DAG.getDataLayout();
     EVT PtrTy = TLI.getPointerTy(DL);
-    SDValue Mode = DAG.getConstant(-1LL, dl, PtrTy);
+    SDValue Mode = DAG.getAllOnesConstant(dl, PtrTy);
     Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
                                                 Node->getOperand(0), dl));
     break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 5b66748297b7ef4..c19a5a4995627a0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3302,7 +3302,7 @@ void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
     SDValue HiNeg =
         DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT);
     if (N->getOpcode() == ISD::SMIN) {
-      Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT));
+      Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getAllOnesConstant(DL, NVT));
     } else {
       Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL);
     }
@@ -4390,7 +4390,7 @@ void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
 
   SDValue SatMax, SatMin;
   SDValue NVTZero = DAG.getConstant(0, dl, NVT);
-  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
+  SDValue NVTNeg1 = DAG.getAllOnesConstant(dl, NVT);
   EVT BoolNVT = getSetCCResultType(NVT);
 
   if (!Signed) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 6de1e3eca7feda8..27dd4ae241bd100 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -865,6 +865,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue ScalarizeVecOp_UnaryOp(SDNode *N);
   SDValue ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N);
   SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);
+  SDValue ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo);
   SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue ScalarizeVecOp_VSELECT(SDNode *N);
   SDValue ScalarizeVecOp_VSETCC(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index ddb7c8c54bbfe46..8c0c2616250c60f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -765,6 +765,9 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::CONCAT_VECTORS:
     Res = ScalarizeVecOp_CONCAT_VECTORS(N);
     break;
+  case ISD::INSERT_SUBVECTOR:
+    Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
+    break;
   case ISD::EXTRACT_VECTOR_ELT:
     Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
     break;
@@ -882,6 +885,19 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
   return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
 }
 
+/// The inserted subvector is to be scalarized - use insert vector element
+/// instead.
+SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
+                                                          unsigned OpNo) {
+  // We should not be attempting to scalarize the containing vector
+  assert(OpNo == 1);
+  SDValue Elt = GetScalarizedVector(N->getOperand(1));
+  SDValue ContainingVec = N->getOperand(0);
+  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
+                     ContainingVec.getValueType(), ContainingVec, Elt,
+                     N->getOperand(2));
+}
+
 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
 /// so just return the element, ignoring the index.
 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/MatchContext.h b/llvm/lib/CodeGen/SelectionDAG/MatchContext.h
index d81a0c2e87e7d28..8f03532af99e865 100644
--- a/llvm/lib/CodeGen/SelectionDAG/MatchContext.h
+++ b/llvm/lib/CodeGen/SelectionDAG/MatchContext.h
@@ -45,6 +45,8 @@ class EmptyMatchContext {
                                 bool LegalOnly = false) const {
     return TLI.isOperationLegalOrCustom(Op, VT, LegalOnly);
   }
+
+  unsigned getNumOperands(SDValue N) const { return N->getNumOperands(); }
 };
 
 class VPMatchContext {
@@ -169,6 +171,10 @@ class VPMatchContext {
     unsigned VPOp = ISD::getVPForBaseOpcode(Op);
     return TLI.isOperationLegalOrCustom(VPOp, VT, LegalOnly);
   }
+
+  unsigned getNumOperands(SDValue N) const {
+    return N->isVPOpcode() ? N->getNumOperands() - 2 : N->getNumOperands();
+  }
 };
 
 } // namespace llvm
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 7bf90ceb93cb4e3..8d6f5334c9600be 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1747,6 +1747,15 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
   return Result;
 }
 
+SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
+                                        bool isT, bool isO) {
+  unsigned Size = VT.getScalarSizeInBits();
+  assert(
+      isIntN(Size, Val) &&
+      "getSignedConstant with a int64_t value that doesn't fit in the type!");
+  return getConstant(APInt(Size, Val, true), DL, VT, isT, isO);
+}
+
 SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
                                          bool IsOpaque) {
   return getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), DL, VT,
@@ -2434,9 +2443,9 @@ SDValue SelectionDAG::expandVAArg(SDNode *Node) {
     VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
                      getConstant(MA->value() - 1, dl, VAList.getValueType()));
 
-    VAList =
-        getNode(ISD::AND, dl, VAList.getValueType(), VAList,
-                getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
+    VAList = getNode(
+        ISD::AND, dl, VAList.getValueType(), VAList,
+        getSignedConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
   }
 
   // Increment the pointer, VAList, to the next vaarg
@@ -7617,16 +7626,34 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
 }
 
 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
-                              SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
+                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
+                              const SDNodeFlags Flags) {
   SDValue Ops[] = { N1, N2, N3, N4 };
-  return getNode(Opcode, DL, VT, Ops);
+  return getNode(Opcode, DL, VT, Ops, Flags);
+}
+
+SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
+                              SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
+  SDNodeFlags Flags;
+  if (Inserter)
+    Flags = Inserter->getFlags();
+  return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
 }
 
 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
-                              SDValue N5) {
+                              SDValue N5, const SDNodeFlags Flags) {
   SDValue Ops[] = { N1, N2, N3, N4, N5 };
-  return getNode(Opcode, DL, VT, Ops);
+  return getNode(Opcode, DL, VT, Ops, Flags);
+}
+
+SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
+                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
+                              SDValue N5) {
+  SDNodeFlags Flags;
+  if (Inserter)
+    Flags = Inserter->getFlags();
+  return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
 }
 
 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 7cdd3d47b641d7b..60dcb1185427856 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -2818,9 +2818,10 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
   //     je foo
   //     cmp D, E
   //     jle foo
+  bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
   const Instruction *BOp = dyn_cast<Instruction>(CondVal);
   if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
-      BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
+      BOp->hasOneUse() && !IsUnpredictable) {
     Value *Vec;
     const Value *BOp0, *BOp1;
     Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
@@ -2869,7 +2870,9 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
 
   // Create a CaseBlock record representing this branch.
   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
-               nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
+               nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
+               BranchProbability::getUnknown(), BranchProbability::getUnknown(),
+               IsUnpredictable);
 
   // Use visitSwitchCase to actually insert the fast branch sequence for this
   // cond branch.
@@ -2957,9 +2960,10 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
   }
 
-  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
-                               MVT::Other, getControlRoot(), Cond,
-                               DAG.getBasicBlock(CB.TrueBB));
+  SDNodeFlags Flags;
+  Flags.setUnpredictable(CB.IsUnpredictable);
+  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
+                               Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
 
   setValue(CurInst, BrCond);
 
@@ -4474,7 +4478,7 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
 
   // Mask out the low bits for alignment purposes.
   AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
-                          DAG.getConstant(~StackAlignMask, dl, IntPtr));
+                          DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
 
   SDValue Ops[] = {
       getRoot(), AllocSize,
@@ -5229,7 +5233,8 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
   // definition.
   const Function *F = I.getCalledFunction();
   bool HasChain = !F->doesNotAccessMemory();
-  bool OnlyLoad = HasChain && F->onlyReadsMemory();
+  bool OnlyLoad =
+      HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
 
   // Build the operand list.
   SmallVector<SDValue, 8> Ops;
@@ -12531,8 +12536,10 @@ void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
 
   // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
   if (VT.isScalableVector()) {
-    setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
-                             DAG.getVectorIdxConstant(Imm, DL)));
+    setValue(
+        &I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
+                        DAG.getSignedConstant(
+                            Imm, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
     return;
   }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 3e517a51862de32..46c24299d1ae97d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -3784,7 +3784,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       if (Opcode >= OPC_EmitInteger && Opcode <= OPC_EmitInteger64)
         Val = decodeSignRotatedValue(Val);
       RecordedNodes.push_back(std::pair<SDValue, SDNode *>(
-          CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch), VT), nullptr));
+          CurDAG->getSignedConstant(Val, SDLoc(NodeToMatch), VT,
+                                    /*isTarget=*/true),
+          nullptr));
       continue;
     }
     case OPC_EmitRegister:
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 2c939967a5e1d9b..4e796289cff0a10 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -598,12 +598,11 @@ bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
 
   // Search for the smallest integer type with free casts to and from
   // Op's type. For expedience, just check power-of-2 integer types.
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   unsigned DemandedSize = DemandedBits.getActiveBits();
   for (unsigned SmallVTBits = llvm::bit_ceil(DemandedSize);
        SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
     EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
-    if (TLI.isTruncateFree(VT, SmallVT) && TLI.isZExtFree(SmallVT, VT)) {
+    if (isTruncateFree(VT, SmallVT) && isZExtFree(SmallVT, VT)) {
       // We found a type with free casts.
       SDValue X = DAG.getNode(
           Op.getOpcode(), dl, SmallVT,
@@ -1622,9 +1621,8 @@ bool TargetLowering::SimplifyDemandedBits(
             APInt Ones = APInt::getAllOnes(BitWidth);
             Ones = Op0Opcode == ISD::SHL ? Ones.shl(ShiftAmt)
                                          : Ones.lshr(ShiftAmt);
-            const TargetLowering &TLI = TLO.DAG.getTargetLoweringInfo();
             if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
-                TLI.isDesirableToCommuteXorWithShift(Op.getNode())) {
+                isDesirableToCommuteXorWithShift(Op.getNode())) {
               // If the xor constant is a demanded mask, do a 'not' before the
               // shift:
               // xor (X << ShiftC), XorC --> (not X) << ShiftC
@@ -3065,8 +3063,7 @@ bool TargetLowering::SimplifyDemandedVectorElts(
 
   KnownUndef = KnownZero = APInt::getZero(NumElts);
 
-  const TargetLowering &TLI = TLO.DAG.getTargetLoweringInfo();
-  if (!TLI.shouldSimplifyDemandedVectorElts(Op, TLO))
+  if (!shouldSimplifyDemandedVectorElts(Op, TLO))
     return false;
 
   // TODO: For now we assume we know nothing about scalable vectors.
@@ -4162,8 +4159,7 @@ SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
 
   // We don't want to do this in every single case.
   SelectionDAG &DAG = DCI.DAG;
-  if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
-          XVT, KeptBits))
+  if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
     return SDValue();
 
   // Unfold into:  sext_inreg(%x) cond %x
@@ -4187,10 +4183,9 @@ SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
   SDValue X, C, Y;
 
   SelectionDAG &DAG = DCI.DAG;
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // Look for '(C l>>/<< Y)'.
-  auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
+  auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
     // The shift should be one-use.
     if (!V.hasOneUse())
       return false;
@@ -4216,7 +4211,7 @@ SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
 
     ConstantSDNode *XC =
         isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
-    return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
+    return shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
         X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
   };
 
@@ -5114,7 +5109,6 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
   // Back to non-vector simplifications.
   // TODO: Can we do these for vector splats?
   if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
-    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     const APInt &C1 = N1C->getAPIntValue();
     EVT ShValTy = N0.getValueType();
 
@@ -5132,7 +5126,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           // Perform the xform if the AND RHS is a single bit.
           unsigned ShCt = AndRHS->getAPIntValue().logBase2();
           if (AndRHS->getAPIntValue().isPowerOf2() &&
-              !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
+              !shouldAvoidTransformToShift(ShValTy, ShCt)) {
             return DAG.getNode(
                 ISD::TRUNCATE, dl, VT,
                 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
@@ -5142,8 +5136,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           // (X & 8) == 8  -->  (X & 8) >> 3
           // Perform the xform if C1 is a single bit.
           unsigned ShCt = C1.logBase2();
-          if (C1.isPowerOf2() &&
-              !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
+          if (C1.isPowerOf2() && !shouldAvoidTransformToShift(ShValTy, ShCt)) {
             return DAG.getNode(
                 ISD::TRUNCATE, dl, VT,
                 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
@@ -5162,7 +5155,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           const APInt &AndRHSC = AndRHS->getAPIntValue();
           if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(AndRHSC)) {
             unsigned ShiftBits = AndRHSC.countr_zero();
-            if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
+            if (!shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
               SDValue Shift = DAG.getNode(
                   ISD::SRL, dl, ShValTy, N0.getOperand(0),
                   DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
@@ -5191,7 +5184,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
         NewC.lshrInPlace(ShiftBits);
         if (ShiftBits && NewC.getSignificantBits() <= 64 &&
             isLegalICmpImmediate(NewC.getSExtValue()) &&
-            !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
+            !shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
           SDValue Shift =
               DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                           DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
@@ -6234,8 +6227,7 @@ SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
                               SelectionDAG &DAG,
                               SmallVectorImpl<SDNode *> &Created) const {
   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
+  if (isIntDivCheap(N->getValueType(0), Attr))
     return SDValue(N, 0); // Lower SDIV as SDIV
   return SDValue();
 }
@@ -6245,8 +6237,7 @@ TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor,
                               SelectionDAG &DAG,
                               SmallVectorImpl<SDNode *> &Created) const {
   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
+  if (isIntDivCheap(N->getValueType(0), Attr))
     return SDValue(N, 0); // Lower SREM as SREM
   return SDValue();
 }
@@ -6358,9 +6349,9 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
     }
 
     MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT));
-    Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
+    Factors.push_back(DAG.getSignedConstant(NumeratorFactor, dl, SVT));
     Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
-    ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
+    ShiftMasks.push_back(DAG.getSignedConstant(ShiftMask, dl, SVT));
     return true;
   };
 
@@ -8526,7 +8517,7 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
 
     // NaN (if exists) will be propagated later, so orderness doesn't matter.
     SDValue Compare =
-        DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETGT : ISD::SETLT);
+        DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETOGT : ISD::SETOLT);
     MinMax = DAG.getSelect(DL, VT, Compare, LHS, RHS, Flags);
   }
 
@@ -8543,7 +8534,7 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
   if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
       !DAG.isKnownNeverZeroFloat(RHS) && !DAG.isKnownNeverZeroFloat(LHS)) {
     SDValue IsZero = DAG.getSetCC(DL, CCVT, MinMax,
-                                  DAG.getConstantFP(0.0, DL, VT), ISD::SETEQ);
+                                  DAG.getConstantFP(0.0, DL, VT), ISD::SETOEQ);
     SDValue TestZero =
         DAG.getTargetConstant(IsMax ? fcPosZero : fcNegZero, DL, MVT::i32);
     SDValue LCmp = DAG.getSelect(
@@ -8832,7 +8823,7 @@ SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op,
   // Split the value into sign bit and absolute value.
   SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ValueMaskV);
   SDValue SignV = DAG.getSetCC(DL, ResultVT, OpAsInt,
-                               DAG.getConstant(0.0, DL, IntVT), ISD::SETLT);
+                               DAG.getConstant(0, DL, IntVT), ISD::SETLT);
 
   // Tests that involve more than one class should be processed first.
   SDValue PartialRes;
@@ -9194,8 +9185,8 @@ SDValue TargetLowering::expandVPCTLZ(SDNode *Node, SelectionDAG &DAG) const {
                      DAG.getNode(ISD::VP_SRL, dl, VT, Op, Tmp, Mask, VL), Mask,
                      VL);
   }
-  Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getConstant(-1, dl, VT), Mask,
-                   VL);
+  Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getAllOnesConstant(dl, VT),
+                   Mask, VL);
   return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL);
 }
 
@@ -9308,7 +9299,7 @@ SDValue TargetLowering::expandVPCTTZ(SDNode *Node, SelectionDAG &DAG) const {
 
   // Same as the vector part of expandCTTZ, use: popcount(~x & (x - 1))
   SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op,
-                            DAG.getConstant(-1, dl, VT), Mask, VL);
+                            DAG.getAllOnesConstant(dl, VT), Mask, VL);
   SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op,
                                  DAG.getConstant(1, dl, VT), Mask, VL);
   SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL);
@@ -10667,7 +10658,7 @@ SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const {
     SDValue SelectZeroOrOne =
         DAG.getSelect(dl, ResVT, IsGT, DAG.getConstant(1, dl, ResVT),
                       DAG.getConstant(0, dl, ResVT));
-    return DAG.getSelect(dl, ResVT, IsLT, DAG.getConstant(-1, dl, ResVT),
+    return DAG.getSelect(dl, ResVT, IsLT, DAG.getAllOnesConstant(dl, ResVT),
                          SelectZeroOrOne);
   }
 
@@ -11746,13 +11737,12 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
                                            SDValue EVL, bool &NeedInvert,
                                            const SDLoc &dl, SDValue &Chain,
                                            bool IsSignaling) const {
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   MVT OpVT = LHS.getSimpleValueType();
   ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
   NeedInvert = false;
   assert(!EVL == !Mask && "VP Mask and EVL must either both be set or unset");
   bool IsNonVP = !EVL;
-  switch (TLI.getCondCodeAction(CCCode, OpVT)) {
+  switch (getCondCodeAction(CCCode, OpVT)) {
   default:
     llvm_unreachable("Unknown condition code action!");
   case TargetLowering::Legal:
@@ -11760,7 +11750,7 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
     break;
   case TargetLowering::Expand: {
     ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
-    if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
+    if (isCondCodeLegalOrCustom(InvCC, OpVT)) {
       std::swap(LHS, RHS);
       CC = DAG.getCondCode(InvCC);
       return true;
@@ -11768,13 +11758,13 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
     // Swapping operands didn't work. Try inverting the condition.
     bool NeedSwap = false;
     InvCC = getSetCCInverse(CCCode, OpVT);
-    if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
+    if (!isCondCodeLegalOrCustom(InvCC, OpVT)) {
       // If inverting the condition is not enough, try swapping operands
       // on top of it.
       InvCC = ISD::getSetCCSwappedOperands(InvCC);
       NeedSwap = true;
     }
-    if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
+    if (isCondCodeLegalOrCustom(InvCC, OpVT)) {
       CC = DAG.getCondCode(InvCC);
       NeedInvert = true;
       if (NeedSwap)
@@ -11788,18 +11778,18 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
     default:
       llvm_unreachable("Don't know how to expand this condition!");
     case ISD::SETUO:
-      if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
+      if (isCondCodeLegal(ISD::SETUNE, OpVT)) {
         CC1 = ISD::SETUNE;
         CC2 = ISD::SETUNE;
         Opc = ISD::OR;
         break;
       }
-      assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
+      assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
              "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
       NeedInvert = true;
       [[fallthrough]];
     case ISD::SETO:
-      assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
+      assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
              "If SETO is expanded, SETOEQ must be legal!");
       CC1 = ISD::SETOEQ;
       CC2 = ISD::SETOEQ;
@@ -11812,9 +11802,8 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
       // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
       // the operands.
       CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
-      if (!TLI.isCondCodeLegal(CC2, OpVT) &&
-          (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||
-           TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {
+      if (!isCondCodeLegal(CC2, OpVT) && (isCondCodeLegal(ISD::SETOGT, OpVT) ||
+                                          isCondCodeLegal(ISD::SETOLT, OpVT))) {
         CC1 = ISD::SETOGT;
         CC2 = ISD::SETOLT;
         Opc = ISD::OR;
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 1b0012b65b80d48..1d52ebe6717f043 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -828,6 +828,8 @@ void TargetPassConfig::addIRPasses() {
     if (!DisableLSR) {
       addPass(createCanonicalizeFreezeInLoopsPass());
       addPass(createLoopStrengthReducePass());
+      if (EnableLoopTermFold)
+        addPass(createLoopTermFoldPass());
       if (PrintLSR)
         addPass(createPrintFunctionPass(dbgs(),
                                         "\n\n*** Code after LSR ***\n"));
diff --git a/llvm/lib/CodeGen/TypePromotion.cpp b/llvm/lib/CodeGen/TypePromotion.cpp
index 11aa3abe9068537..b29c46b0540cdce 100644
--- a/llvm/lib/CodeGen/TypePromotion.cpp
+++ b/llvm/lib/CodeGen/TypePromotion.cpp
@@ -665,8 +665,8 @@ void IRPromoter::Mutate() {
     } else if (auto *Switch = dyn_cast<SwitchInst>(I))
       TruncTysMap[I].push_back(Switch->getCondition()->getType());
     else {
-      for (unsigned i = 0; i < I->getNumOperands(); ++i)
-        TruncTysMap[I].push_back(I->getOperand(i)->getType());
+      for (const Value *Op : I->operands())
+        TruncTysMap[I].push_back(Op->getType());
     }
   }
   for (auto *V : Visited) {
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index f36605bbf03baf7..7565b4762bb9f7c 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -29,8 +29,10 @@ add_llvm_component_library(LLVMOrcJIT
   JITTargetMachineBuilder.cpp
   LazyReexports.cpp
   Layer.cpp
+  LoadRelocatableObject.cpp
   LookupAndRecordAddrs.cpp
   LLJIT.cpp
+  MachO.cpp
   MachOPlatform.cpp
   MapperJITLinkMemoryManager.cpp
   MemoryMapper.cpp
@@ -60,6 +62,7 @@ add_llvm_component_library(LLVMOrcJIT
   ${atomic_lib}
 
   LINK_COMPONENTS
+  BinaryFormat
   Core
   ExecutionEngine
   JITLink
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index c1a193f6a280248..36bd425beab8de8 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -9,6 +9,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/ExecutionEngine/Orc/MachO.h"
 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
@@ -295,7 +296,7 @@ StaticLibraryDefinitionGenerator::Load(
 
     const auto &TT = L.getExecutionSession().getTargetTriple();
 
-    auto SliceRange = getSliceRangeForArch(*UB, TT);
+    auto SliceRange = getMachOSliceRangeForTriple(*UB, TT);
     if (!SliceRange)
       return SliceRange.takeError();
 
@@ -358,7 +359,7 @@ StaticLibraryDefinitionGenerator::Create(
 
     const auto &TT = L.getExecutionSession().getTargetTriple();
 
-    auto SliceRange = getSliceRangeForArch(*UB, TT);
+    auto SliceRange = getMachOSliceRangeForTriple(*UB, TT);
     if (!SliceRange)
       return SliceRange.takeError();
 
@@ -460,27 +461,6 @@ Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() {
   return Error::success();
 }
 
-Expected<std::pair<size_t, size_t>>
-StaticLibraryDefinitionGenerator::getSliceRangeForArch(
-    object::MachOUniversalBinary &UB, const Triple &TT) {
-
-  for (const auto &Obj : UB.objects()) {
-    auto ObjTT = Obj.getTriple();
-    if (ObjTT.getArch() == TT.getArch() &&
-        ObjTT.getSubArch() == TT.getSubArch() &&
-        (TT.getVendor() == Triple::UnknownVendor ||
-         ObjTT.getVendor() == TT.getVendor())) {
-      // We found a match. Return the range for the slice.
-      return std::make_pair(Obj.getOffset(), Obj.getSize());
-    }
-  }
-
-  return make_error<StringError>(Twine("Universal binary ") + UB.getFileName() +
-                                     " does not contain a slice for " +
-                                     TT.str(),
-                                 inconvertibleErrorCode());
-}
-
 StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator(
     ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,
     std::unique_ptr<object::Archive> Archive,
diff --git a/llvm/lib/ExecutionEngine/Orc/LoadRelocatableObject.cpp b/llvm/lib/ExecutionEngine/Orc/LoadRelocatableObject.cpp
new file mode 100644
index 000000000000000..5fde1b5fb4da927
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/LoadRelocatableObject.cpp
@@ -0,0 +1,69 @@
+//===----- LoadRelocatableObject.cpp -- Load relocatable object files -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/LoadRelocatableObject.h"
+#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/ExecutionEngine/Orc/MachO.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm {
+namespace orc {
+
+static Expected<std::unique_ptr<MemoryBuffer>>
+checkCOFFRelocatableObject(std::unique_ptr<MemoryBuffer> Obj,
+                           const Triple &TT) {
+  // TODO: Actually check the architecture of the file.
+  return std::move(Obj);
+}
+
+static Expected<std::unique_ptr<MemoryBuffer>>
+checkELFRelocatableObject(std::unique_ptr<MemoryBuffer> Obj, const Triple &TT) {
+  // TODO: Actually check the architecture of the file.
+  return std::move(Obj);
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+loadRelocatableObject(StringRef Path, const Triple &TT) {
+  auto Buf = MemoryBuffer::getFile(Path);
+  if (!Buf)
+    return createFileError(Path, Buf.getError());
+
+  std::optional<Triple::ObjectFormatType> RequireFormat;
+  if (TT.getObjectFormat() != Triple::UnknownObjectFormat)
+    RequireFormat = TT.getObjectFormat();
+
+  switch (identify_magic((*Buf)->getBuffer())) {
+  case file_magic::coff_object:
+    if (!RequireFormat || *RequireFormat == Triple::COFF)
+      return checkCOFFRelocatableObject(std::move(*Buf), TT);
+    break;
+  case file_magic::elf_relocatable:
+    if (!RequireFormat || *RequireFormat == Triple::ELF)
+      return checkELFRelocatableObject(std::move(*Buf), TT);
+    break;
+  case file_magic::macho_object:
+    if (!RequireFormat || *RequireFormat == Triple::MachO)
+      return checkMachORelocatableObject(std::move(*Buf), TT, false);
+    break;
+  case file_magic::macho_universal_binary:
+    if (!RequireFormat || *RequireFormat == Triple::MachO)
+      return loadMachORelocatableObjectFromUniversalBinary(Path,
+                                                           std::move(*Buf), TT);
+    break;
+  default:
+    break;
+  }
+  return make_error<StringError>(
+      Path + " does not contain a relocatable object file compatible with " +
+          TT.str(),
+      inconvertibleErrorCode());
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/llvm/lib/ExecutionEngine/Orc/MachO.cpp b/llvm/lib/ExecutionEngine/Orc/MachO.cpp
new file mode 100644
index 000000000000000..fe5a1dbbb9f8a1a
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/MachO.cpp
@@ -0,0 +1,164 @@
+//===----------------- MachO.cpp - MachO format utilities -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/MachO.h"
+
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Object/MachOUniversal.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm {
+namespace orc {
+
+static std::string objDesc(MemoryBuffer &Obj, const Triple &TT,
+                           bool ObjIsSlice) {
+  std::string Desc;
+  if (ObjIsSlice)
+    Desc += (TT.getArchName() + " slice of universal binary").str();
+  Desc += Obj.getBufferIdentifier();
+  return Desc;
+}
+
+template <typename HeaderType>
+static Expected<std::unique_ptr<MemoryBuffer>>
+checkMachORelocatableObject(std::unique_ptr<MemoryBuffer> Obj,
+                            bool SwapEndianness, const Triple &TT,
+                            bool ObjIsSlice) {
+  StringRef Data = Obj->getBuffer();
+
+  HeaderType Hdr;
+  memcpy(&Hdr, Data.data(), sizeof(HeaderType));
+
+  if (SwapEndianness)
+    swapStruct(Hdr);
+
+  if (Hdr.filetype != MachO::MH_OBJECT)
+    return make_error<StringError>(objDesc(*Obj, TT, ObjIsSlice) +
+                                       " is not a MachO relocatable object",
+                                   inconvertibleErrorCode());
+
+  auto ObjArch = object::MachOObjectFile::getArch(Hdr.cputype, Hdr.cpusubtype);
+  if (ObjArch != TT.getArch())
+    return make_error<StringError>(
+        objDesc(*Obj, TT, ObjIsSlice) + Triple::getArchTypeName(ObjArch) +
+            ", cannot be loaded into " + TT.str() + " process",
+        inconvertibleErrorCode());
+
+  return std::move(Obj);
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+checkMachORelocatableObject(std::unique_ptr<MemoryBuffer> Obj, const Triple &TT,
+                            bool ObjIsSlice) {
+  StringRef Data = Obj->getBuffer();
+
+  if (Data.size() < 4)
+    return make_error<StringError>(
+        objDesc(*Obj, TT, ObjIsSlice) +
+            " is not a valid MachO relocatable object file (truncated header)",
+        inconvertibleErrorCode());
+
+  uint32_t Magic;
+  memcpy(&Magic, Data.data(), sizeof(uint32_t));
+
+  switch (Magic) {
+  case MachO::MH_MAGIC:
+  case MachO::MH_CIGAM:
+    return checkMachORelocatableObject<MachO::mach_header>(
+        std::move(Obj), Magic == MachO::MH_CIGAM, TT, ObjIsSlice);
+  case MachO::MH_MAGIC_64:
+  case MachO::MH_CIGAM_64:
+    return checkMachORelocatableObject<MachO::mach_header_64>(
+        std::move(Obj), Magic == MachO::MH_CIGAM_64, TT, ObjIsSlice);
+  default:
+    return make_error<StringError>(
+        objDesc(*Obj, TT, ObjIsSlice) +
+            " is not a valid MachO relocatable object (bad magic value)",
+        inconvertibleErrorCode());
+  }
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+loadMachORelocatableObject(StringRef Path, const Triple &TT) {
+  assert((TT.getObjectFormat() == Triple::UnknownObjectFormat ||
+          TT.getObjectFormat() == Triple::MachO) &&
+         "TT must specify MachO or Unknown object format");
+
+  auto Buf = MemoryBuffer::getFile(Path);
+  if (!Buf)
+    return createFileError(Path, Buf.getError());
+
+  switch (identify_magic((*Buf)->getBuffer())) {
+  case file_magic::macho_object:
+    return checkMachORelocatableObject(std::move(*Buf), TT, false);
+  case file_magic::macho_universal_binary:
+    return loadMachORelocatableObjectFromUniversalBinary(Path, std::move(*Buf),
+                                                         TT);
+  default:
+    return make_error<StringError>(
+        Path + " does not contain a relocatable object file compatible with " +
+            TT.str(),
+        inconvertibleErrorCode());
+  }
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+loadMachORelocatableObjectFromUniversalBinary(
+    StringRef UBPath, std::unique_ptr<MemoryBuffer> UBBuf, const Triple &TT) {
+
+  auto UniversalBin =
+      object::MachOUniversalBinary::create(UBBuf->getMemBufferRef());
+  if (!UniversalBin)
+    return UniversalBin.takeError();
+
+  auto SliceRange = getMachOSliceRangeForTriple(**UniversalBin, TT);
+  if (!SliceRange)
+    return SliceRange.takeError();
+
+  auto ObjBuf = errorOrToExpected(MemoryBuffer::getFileSlice(
+      UBPath, SliceRange->second, SliceRange->first, false));
+  if (!ObjBuf)
+    return createFileError(UBPath, ObjBuf.takeError());
+
+  return checkMachORelocatableObject(std::move(*ObjBuf), TT, true);
+}
+
+Expected<std::pair<size_t, size_t>>
+getMachOSliceRangeForTriple(object::MachOUniversalBinary &UB,
+                            const Triple &TT) {
+
+  for (const auto &Obj : UB.objects()) {
+    auto ObjTT = Obj.getTriple();
+    if (ObjTT.getArch() == TT.getArch() &&
+        ObjTT.getSubArch() == TT.getSubArch() &&
+        (TT.getVendor() == Triple::UnknownVendor ||
+         ObjTT.getVendor() == TT.getVendor())) {
+      // We found a match. Return the range for the slice.
+      return std::make_pair(Obj.getOffset(), Obj.getSize());
+    }
+  }
+
+  return make_error<StringError>(Twine("Universal binary ") + UB.getFileName() +
+                                     " does not contain a slice for " +
+                                     TT.str(),
+                                 inconvertibleErrorCode());
+}
+
+Expected<std::pair<size_t, size_t>>
+getMachOSliceRangeForTriple(MemoryBufferRef UBBuf, const Triple &TT) {
+
+  auto UB = object::MachOUniversalBinary::create(UBBuf);
+  if (!UB)
+    return UB.takeError();
+
+  return getMachOSliceRangeForTriple(**UB, TT);
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index f9b070e6f1eae4e..70a6e74b94d55cf 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
+#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
@@ -39,8 +40,8 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/PassManager.h"
 #include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/IR/ReplaceConstant.h"
 #include "llvm/IR/Value.h"
 #include "llvm/MC/TargetRegistry.h"
@@ -501,15 +502,20 @@ void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
   Value *ZeroArray = Constant::getNullValue(ArrayType::get(Int32Ty, MaxDim));
   Value *Flags = Builder.getInt64(KernelArgs.HasNoWait);
 
-  assert(!KernelArgs.NumTeams.empty());
+  assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());
 
   Value *NumTeams3D =
       Builder.CreateInsertValue(ZeroArray, KernelArgs.NumTeams[0], {0});
-  for (unsigned I = 1; I < std::min(KernelArgs.NumTeams.size(), MaxDim); ++I)
+  Value *NumThreads3D =
+      Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads[0], {0});
+  for (unsigned I :
+       seq<unsigned>(1, std::min(KernelArgs.NumTeams.size(), MaxDim)))
     NumTeams3D =
         Builder.CreateInsertValue(NumTeams3D, KernelArgs.NumTeams[I], {I});
-  Value *NumThreads3D =
-      Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads, {0});
+  for (unsigned I :
+       seq<unsigned>(1, std::min(KernelArgs.NumThreads.size(), MaxDim)))
+    NumThreads3D =
+        Builder.CreateInsertValue(NumThreads3D, KernelArgs.NumThreads[I], {I});
 
   ArgsVector = {Version,
                 PointerNum,
@@ -1114,9 +1120,9 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitKernelLaunch(
   // __tgt_target_teams() launches a GPU kernel with the requested number
   // of teams and threads so no additional calls to the runtime are required.
   // Check the error code and execute the host version if required.
-  Builder.restoreIP(emitTargetKernel(Builder, AllocaIP, Return, RTLoc, DeviceID,
-                                     Args.NumTeams.front(), Args.NumThreads,
-                                     OutlinedFnID, ArgsVector));
+  Builder.restoreIP(emitTargetKernel(
+      Builder, AllocaIP, Return, RTLoc, DeviceID, Args.NumTeams.front(),
+      Args.NumThreads.front(), OutlinedFnID, ArgsVector));
 
   BasicBlock *OffloadFailedBlock =
       BasicBlock::Create(Builder.getContext(), "omp_offload.failed");
@@ -7075,8 +7081,8 @@ void OpenMPIRBuilder::emitOffloadingArraysAndArgs(
 static void emitTargetCall(
     OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
     OpenMPIRBuilder::InsertPointTy AllocaIP, Function *OutlinedFn,
-    Constant *OutlinedFnID, ArrayRef<int32_t> NumTeams, int32_t NumThreads,
-    SmallVectorImpl<Value *> &Args,
+    Constant *OutlinedFnID, ArrayRef<int32_t> NumTeams,
+    ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Args,
     OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
     SmallVector<llvm::OpenMPIRBuilder::DependData> Dependencies = {}) {
   // Generate a function call to the host fallback implementation of the target
@@ -7123,13 +7129,15 @@ static void emitTargetCall(
                                          /*ForEndCall=*/false);
 
   SmallVector<Value *, 3> NumTeamsC;
+  SmallVector<Value *, 3> NumThreadsC;
   for (auto V : NumTeams)
     NumTeamsC.push_back(llvm::ConstantInt::get(Builder.getInt32Ty(), V));
+  for (auto V : NumThreads)
+    NumThreadsC.push_back(llvm::ConstantInt::get(Builder.getInt32Ty(), V));
 
   unsigned NumTargetItems = Info.NumberOfPtrs;
   // TODO: Use correct device ID
   Value *DeviceID = Builder.getInt64(OMP_DEVICEID_UNDEF);
-  Value *NumThreadsVal = Builder.getInt32(NumThreads);
   uint32_t SrcLocStrSize;
   Constant *SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
   Value *RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
@@ -7140,8 +7148,8 @@ static void emitTargetCall(
   Value *DynCGGroupMem = Builder.getInt32(0);
 
   OpenMPIRBuilder::TargetKernelArgs KArgs(NumTargetItems, RTArgs, NumIterations,
-                                          NumTeamsC, NumThreadsVal,
-                                          DynCGGroupMem, HasNoWait);
+                                          NumTeamsC, NumThreadsC, DynCGGroupMem,
+                                          HasNoWait);
 
   // The presence of certain clauses on the target directive require the
   // explicit generation of the target task.
@@ -7159,11 +7167,11 @@ static void emitTargetCall(
 OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
     const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
     InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo,
-    ArrayRef<int32_t> NumTeams, int32_t NumThreads,
+    ArrayRef<int32_t> NumTeams, ArrayRef<int32_t> NumThreads,
     SmallVectorImpl<Value *> &Args, GenMapInfoCallbackTy GenMapInfoCB,
     OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
     OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
-    SmallVector<DependData> Dependenciess) {
+    SmallVector<DependData> Dependencies) {
 
   if (!updateToLocation(Loc))
     return InsertPointTy();
@@ -7184,7 +7192,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
   // that represents the target region. Do that now.
   if (!Config.isTargetDevice())
     emitTargetCall(*this, Builder, AllocaIP, OutlinedFn, OutlinedFnID, NumTeams,
-                   NumThreads, Args, GenMapInfoCB, Dependenciess);
+                   NumThreads, Args, GenMapInfoCB, Dependencies);
   return Builder.saveIP();
 }
 
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 17c0bf72ef05dc3..dcad76ee8491dd8 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -2203,6 +2203,11 @@ LLVMValueRef LLVMGetNamedGlobal(LLVMModuleRef M, const char *Name) {
   return wrap(unwrap(M)->getNamedGlobal(Name));
 }
 
+LLVMValueRef LLVMGetNamedGlobalWithLength(LLVMModuleRef M, const char *Name,
+                                          size_t Length) {
+  return wrap(unwrap(M)->getNamedGlobal(StringRef(Name, Length)));
+}
+
 LLVMValueRef LLVMGetFirstGlobal(LLVMModuleRef M) {
   Module *Mod = unwrap(M);
   Module::global_iterator I = Mod->global_begin();
@@ -2381,6 +2386,11 @@ LLVMValueRef LLVMGetNamedFunction(LLVMModuleRef M, const char *Name) {
   return wrap(unwrap(M)->getFunction(Name));
 }
 
+LLVMValueRef LLVMGetNamedFunctionWithLength(LLVMModuleRef M, const char *Name,
+                                            size_t Length) {
+  return wrap(unwrap(M)->getFunction(StringRef(Name, Length)));
+}
+
 LLVMValueRef LLVMGetFirstFunction(LLVMModuleRef M) {
   Module *Mod = unwrap(M);
   Module::iterator I = Mod->begin();
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 44cd1e69818953d..5efdaaccf1fc7fc 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -17,6 +17,7 @@
 
 #include "llvm/IR/DataLayout.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -153,6 +154,24 @@ bool DataLayout::PointerSpec::operator==(const PointerSpec &Other) const {
          IndexBitWidth == Other.IndexBitWidth;
 }
 
+namespace {
+/// Predicate to sort primitive specs by bit width.
+struct LessPrimitiveBitWidth {
+  bool operator()(const DataLayout::PrimitiveSpec &LHS,
+                  unsigned RHSBitWidth) const {
+    return LHS.BitWidth < RHSBitWidth;
+  }
+};
+
+/// Predicate to sort pointer specs by address space number.
+struct LessPointerAddrSpace {
+  bool operator()(const DataLayout::PointerSpec &LHS,
+                  unsigned RHSAddrSpace) const {
+    return LHS.AddrSpace < RHSAddrSpace;
+  }
+};
+} // namespace
+
 const char *DataLayout::getManglingComponent(const Triple &T) {
   if (T.isOSBinFormatGOFF())
     return "-m:l";
@@ -197,7 +216,7 @@ DataLayout::DataLayout()
       PointerSpecs(ArrayRef(DefaultPointerSpecs)) {}
 
 DataLayout::DataLayout(StringRef LayoutString) : DataLayout() {
-  if (Error Err = parseSpecifier(LayoutString))
+  if (Error Err = parseLayoutString(LayoutString))
     report_fatal_error(std::move(Err));
 }
 
@@ -242,9 +261,9 @@ bool DataLayout::operator==(const DataLayout &Other) const {
          StructPrefAlignment == Other.StructPrefAlignment;
 }
 
-Expected<DataLayout> DataLayout::parse(StringRef LayoutDescription) {
+Expected<DataLayout> DataLayout::parse(StringRef LayoutString) {
   DataLayout Layout;
-  if (Error Err = Layout.parseSpecifier(LayoutDescription))
+  if (Error Err = Layout.parseLayoutString(LayoutString))
     return std::move(Err);
   return Layout;
 }
@@ -253,6 +272,70 @@ static Error reportError(const Twine &Message) {
   return createStringError(inconvertibleErrorCode(), Message);
 }
 
+static Error createSpecFormatError(Twine Format) {
+  return createStringError("malformed specification, must be of the form \"" +
+                           Format + "\"");
+}
+
+/// Attempts to parse an address space component of a specification.
+static Error parseAddrSpace(StringRef Str, unsigned &AddrSpace) {
+  if (Str.empty())
+    return createStringError("address space component cannot be empty");
+
+  if (!to_integer(Str, AddrSpace, 10) || !isUInt<24>(AddrSpace))
+    return createStringError("address space must be a 24-bit integer");
+
+  return Error::success();
+}
+
+/// Attempts to parse a size component of a specification.
+static Error parseSize(StringRef Str, unsigned &BitWidth,
+                       StringRef Name = "size") {
+  if (Str.empty())
+    return createStringError(Name + " component cannot be empty");
+
+  if (!to_integer(Str, BitWidth, 10) || BitWidth == 0 || !isUInt<24>(BitWidth))
+    return createStringError(Name + " must be a non-zero 24-bit integer");
+
+  return Error::success();
+}
+
+/// Attempts to parse an alignment component of a specification.
+///
+/// On success, returns the value converted to byte amount in \p Alignment.
+/// If the value is zero and \p AllowZero is true, \p Alignment is set to one.
+///
+/// Return an error in a number of cases:
+/// - \p Str is empty or contains characters other than decimal digits;
+/// - the value is zero and \p AllowZero is false;
+/// - the value is too large;
+/// - the value is not a multiple of the byte width;
+/// - the value converted to byte amount is not not a power of two.
+static Error parseAlignment(StringRef Str, Align &Alignment, StringRef Name,
+                            bool AllowZero = false) {
+  if (Str.empty())
+    return createStringError(Name + " alignment component cannot be empty");
+
+  unsigned Value;
+  if (!to_integer(Str, Value, 10) || !isUInt<16>(Value))
+    return createStringError(Name + " alignment must be a 16-bit integer");
+
+  if (Value == 0) {
+    if (!AllowZero)
+      return createStringError(Name + " alignment must be non-zero");
+    Alignment = Align(1);
+    return Error::success();
+  }
+
+  constexpr unsigned ByteWidth = 8;
+  if (Value % ByteWidth || !isPowerOf2_32(Value / ByteWidth))
+    return createStringError(
+        Name + " alignment must be a power of two times the byte width");
+
+  Alignment = Align(Value / ByteWidth);
+  return Error::success();
+}
+
 /// Checked version of split, to ensure mandatory subparts.
 static Error split(StringRef Str, char Separator,
                    std::pair<StringRef, StringRef> &Split) {
@@ -293,336 +376,326 @@ static Error getAddrSpace(StringRef R, unsigned &AddrSpace) {
   return Error::success();
 }
 
-Error DataLayout::parseSpecifier(StringRef Desc) {
-  StringRepresentation = std::string(Desc);
-  while (!Desc.empty()) {
-    // Split at '-'.
-    std::pair<StringRef, StringRef> Split;
-    if (Error Err = ::split(Desc, '-', Split))
+Error DataLayout::parsePrimitiveSpec(StringRef Spec) {
+  // [ifv]<size>:<abi>[:<pref>]
+  SmallVector<StringRef, 3> Components;
+  char Specifier = Spec.front();
+  assert(Specifier == 'i' || Specifier == 'f' || Specifier == 'v');
+  Spec.drop_front().split(Components, ':');
+
+  if (Components.size() < 2 || Components.size() > 3)
+    return createSpecFormatError(Twine(Specifier) + "<size>:<abi>[:<pref>]");
+
+  // Size. Required, cannot be zero.
+  unsigned BitWidth;
+  if (Error Err = parseSize(Components[0], BitWidth))
+    return Err;
+
+  // ABI alignment.
+  Align ABIAlign;
+  if (Error Err = parseAlignment(Components[1], ABIAlign, "ABI"))
+    return Err;
+
+  if (Specifier == 'i' && BitWidth == 8 && ABIAlign != 1)
+    return createStringError("i8 must be 8-bit aligned");
+
+  // Preferred alignment. Optional, defaults to the ABI alignment.
+  Align PrefAlign = ABIAlign;
+  if (Components.size() > 2)
+    if (Error Err = parseAlignment(Components[2], PrefAlign, "preferred"))
       return Err;
-    Desc = Split.second;
 
-    // Split at ':'.
-    if (Error Err = ::split(Split.first, ':', Split))
+  if (PrefAlign < ABIAlign)
+    return createStringError(
+        "preferred alignment cannot be less than the ABI alignment");
+
+  setPrimitiveSpec(Specifier, BitWidth, ABIAlign, PrefAlign);
+  return Error::success();
+}
+
+Error DataLayout::parseAggregateSpec(StringRef Spec) {
+  // a<size>:<abi>[:<pref>]
+  SmallVector<StringRef, 3> Components;
+  assert(Spec.front() == 'a');
+  Spec.drop_front().split(Components, ':');
+
+  if (Components.size() < 2 || Components.size() > 3)
+    return createSpecFormatError("a:<abi>[:<pref>]");
+
+  // According to LangRef, <size> component must be absent altogether.
+  // For backward compatibility, allow it to be specified, but require
+  // it to be zero.
+  if (!Components[0].empty()) {
+    unsigned BitWidth;
+    if (!to_integer(Components[0], BitWidth, 10) || BitWidth != 0)
+      return createStringError("size must be zero");
+  }
+
+  // ABI alignment. Required. Can be zero, meaning use one byte alignment.
+  Align ABIAlign;
+  if (Error Err =
+          parseAlignment(Components[1], ABIAlign, "ABI", /*AllowZero=*/true))
+    return Err;
+
+  // Preferred alignment. Optional, defaults to the ABI alignment.
+  Align PrefAlign = ABIAlign;
+  if (Components.size() > 2)
+    if (Error Err = parseAlignment(Components[2], PrefAlign, "preferred"))
       return Err;
 
-    // Aliases used below.
-    StringRef &Tok  = Split.first;  // Current token.
-    StringRef &Rest = Split.second; // The rest of the string.
-
-    if (Tok == "ni") {
-      do {
-        if (Error Err = ::split(Rest, ':', Split))
-          return Err;
-        Rest = Split.second;
-        unsigned AS;
-        if (Error Err = getInt(Split.first, AS))
-          return Err;
-        if (AS == 0)
-          return reportError("Address space 0 can never be non-integral");
-        NonIntegralAddressSpaces.push_back(AS);
-      } while (!Rest.empty());
-
-      continue;
-    }
+  if (PrefAlign < ABIAlign)
+    return createStringError(
+        "preferred alignment cannot be less than the ABI alignment");
 
-    char SpecifierChar = Tok.front();
-    Tok = Tok.substr(1);
+  StructABIAlignment = ABIAlign;
+  StructPrefAlignment = PrefAlign;
+  return Error::success();
+}
 
-    switch (SpecifierChar) {
-    case 's':
-      // Deprecated, but ignoring here to preserve loading older textual llvm
-      // ASM file
-      break;
-    case 'E':
-      BigEndian = true;
-      break;
-    case 'e':
-      BigEndian = false;
-      break;
-    case 'p': {
-      // Address space.
-      unsigned AddrSpace = 0;
-      if (!Tok.empty())
-        if (Error Err = getInt(Tok, AddrSpace))
-          return Err;
-      if (!isUInt<24>(AddrSpace))
-        return reportError("Invalid address space, must be a 24-bit integer");
-
-      // Size.
-      if (Rest.empty())
-        return reportError(
-            "Missing size specification for pointer in datalayout string");
-      if (Error Err = ::split(Rest, ':', Split))
-        return Err;
-      unsigned PointerMemSize;
-      if (Error Err = getInt(Tok, PointerMemSize))
-        return Err;
-      if (!PointerMemSize)
-        return reportError("Invalid pointer size of 0 bytes");
+Error DataLayout::parsePointerSpec(StringRef Spec) {
+  // p[<n>]:<size>:<abi>[:<pref>[:<idx>]]
+  SmallVector<StringRef, 5> Components;
+  assert(Spec.front() == 'p');
+  Spec.drop_front().split(Components, ':');
 
-      // ABI alignment.
-      if (Rest.empty())
-        return reportError(
-            "Missing alignment specification for pointer in datalayout string");
-      if (Error Err = ::split(Rest, ':', Split))
-        return Err;
-      unsigned PointerABIAlign;
-      if (Error Err = getIntInBytes(Tok, PointerABIAlign))
-        return Err;
-      if (!isPowerOf2_64(PointerABIAlign))
-        return reportError("Pointer ABI alignment must be a power of 2");
-
-      // Size of index used in GEP for address calculation.
-      // The parameter is optional. By default it is equal to size of pointer.
-      unsigned IndexSize = PointerMemSize;
-
-      // Preferred alignment.
-      unsigned PointerPrefAlign = PointerABIAlign;
-      if (!Rest.empty()) {
-        if (Error Err = ::split(Rest, ':', Split))
-          return Err;
-        if (Error Err = getIntInBytes(Tok, PointerPrefAlign))
-          return Err;
-        if (!isPowerOf2_64(PointerPrefAlign))
-          return reportError(
-              "Pointer preferred alignment must be a power of 2");
-
-        // Now read the index. It is the second optional parameter here.
-        if (!Rest.empty()) {
-          if (Error Err = ::split(Rest, ':', Split))
-            return Err;
-          if (Error Err = getInt(Tok, IndexSize))
-            return Err;
-          if (!IndexSize)
-            return reportError("Invalid index size of 0 bytes");
-        }
-      }
-      if (Error Err = setPointerSpec(
-              AddrSpace, PointerMemSize, assumeAligned(PointerABIAlign),
-              assumeAligned(PointerPrefAlign), IndexSize))
+  if (Components.size() < 3 || Components.size() > 5)
+    return createSpecFormatError("p[<n>]:<size>:<abi>[:<pref>[:<idx>]]");
+
+  // Address space. Optional, defaults to 0.
+  unsigned AddrSpace = 0;
+  if (!Components[0].empty())
+    if (Error Err = parseAddrSpace(Components[0], AddrSpace))
+      return Err;
+
+  // Size. Required, cannot be zero.
+  unsigned BitWidth;
+  if (Error Err = parseSize(Components[1], BitWidth, "pointer size"))
+    return Err;
+
+  // ABI alignment. Required, cannot be zero.
+  Align ABIAlign;
+  if (Error Err = parseAlignment(Components[2], ABIAlign, "ABI"))
+    return Err;
+
+  // Preferred alignment. Optional, defaults to the ABI alignment.
+  // Cannot be zero.
+  Align PrefAlign = ABIAlign;
+  if (Components.size() > 3)
+    if (Error Err = parseAlignment(Components[3], PrefAlign, "preferred"))
+      return Err;
+
+  if (PrefAlign < ABIAlign)
+    return createStringError(
+        "preferred alignment cannot be less than the ABI alignment");
+
+  // Index size. Optional, defaults to pointer size. Cannot be zero.
+  unsigned IndexBitWidth = BitWidth;
+  if (Components.size() > 4)
+    if (Error Err = parseSize(Components[4], IndexBitWidth, "index size"))
+      return Err;
+
+  if (IndexBitWidth > BitWidth)
+    return createStringError(
+        "index size cannot be larger than the pointer size");
+
+  setPointerSpec(AddrSpace, BitWidth, ABIAlign, PrefAlign, IndexBitWidth);
+  return Error::success();
+}
+
+Error DataLayout::parseSpecification(StringRef Spec) {
+  // The "ni" specifier is the only two-character specifier. Handle it first.
+  if (Spec.starts_with("ni")) {
+    // ni:<address space>[:<address space>]...
+    StringRef Rest = Spec.drop_front(2);
+
+    // Drop the first ':', then split the rest of the string the usual way.
+    if (!Rest.consume_front(":"))
+      return createSpecFormatError("ni:<address space>[:<address space>]...");
+
+    for (StringRef Str : split(Rest, ':')) {
+      unsigned AddrSpace;
+      if (Error Err = parseAddrSpace(Str, AddrSpace))
         return Err;
-      break;
+      if (AddrSpace == 0)
+        return createStringError("address space 0 cannot be non-integral");
+      NonIntegralAddressSpaces.push_back(AddrSpace);
     }
-    case 'i':
-    case 'v':
-    case 'f':
-    case 'a': {
-      TypeSpecifier Specifier;
-      switch (SpecifierChar) {
-      default:
-        llvm_unreachable("Unexpected specifier!");
-      case 'i':
-        Specifier = TypeSpecifier::Integer;
-        break;
-      case 'v':
-        Specifier = TypeSpecifier::Vector;
-        break;
-      case 'f':
-        Specifier = TypeSpecifier::Float;
-        break;
-      case 'a':
-        Specifier = TypeSpecifier::Aggregate;
-        break;
-      }
+    return Error::success();
+  }
 
-      // Bit size.
-      unsigned Size = 0;
-      if (!Tok.empty())
-        if (Error Err = getInt(Tok, Size))
-          return Err;
+  // The rest of the specifiers are single-character.
+  assert(!Spec.empty() && "Empty specification is handled by the caller");
+  char Specifier = Spec.front();
 
-      if (Specifier == TypeSpecifier::Aggregate && Size != 0)
-        return reportError(
-            "Sized aggregate specification in datalayout string");
+  if (Specifier == 'i' || Specifier == 'f' || Specifier == 'v')
+    return parsePrimitiveSpec(Spec);
 
-      // ABI alignment.
-      if (Rest.empty())
-        return reportError(
-            "Missing alignment specification in datalayout string");
-      if (Error Err = ::split(Rest, ':', Split))
-        return Err;
-      unsigned ABIAlign;
-      if (Error Err = getIntInBytes(Tok, ABIAlign))
-        return Err;
-      if (Specifier != TypeSpecifier::Aggregate && !ABIAlign)
-        return reportError(
-            "ABI alignment specification must be >0 for non-aggregate types");
+  if (Specifier == 'a')
+    return parseAggregateSpec(Spec);
 
-      if (!isUInt<16>(ABIAlign))
-        return reportError("Invalid ABI alignment, must be a 16bit integer");
-      if (ABIAlign != 0 && !isPowerOf2_64(ABIAlign))
-        return reportError("Invalid ABI alignment, must be a power of 2");
-      if (Specifier == TypeSpecifier::Integer && Size == 8 && ABIAlign != 1)
-        return reportError(
-            "Invalid ABI alignment, i8 must be naturally aligned");
-
-      // Preferred alignment.
-      unsigned PrefAlign = ABIAlign;
-      if (!Rest.empty()) {
-        if (Error Err = ::split(Rest, ':', Split))
-          return Err;
-        if (Error Err = getIntInBytes(Tok, PrefAlign))
-          return Err;
-      }
-
-      if (!isUInt<16>(PrefAlign))
-        return reportError(
-            "Invalid preferred alignment, must be a 16bit integer");
-      if (PrefAlign != 0 && !isPowerOf2_64(PrefAlign))
-        return reportError("Invalid preferred alignment, must be a power of 2");
+  if (Specifier == 'p')
+    return parsePointerSpec(Spec);
 
-      if (Error Err = setPrimitiveSpec(Specifier, Size, assumeAligned(ABIAlign),
-                                       assumeAligned(PrefAlign)))
-        return Err;
+  // Split at ':'.
+  std::pair<StringRef, StringRef> Split;
+  if (Error Err = ::split(Spec, ':', Split))
+    return Err;
 
-      break;
-    }
-    case 'n':  // Native integer types.
-      while (true) {
-        unsigned Width;
-        if (Error Err = getInt(Tok, Width))
-          return Err;
-        if (Width == 0)
-          return reportError(
-              "Zero width native integer type in datalayout string");
-        LegalIntWidths.push_back(Width);
-        if (Rest.empty())
-          break;
-        if (Error Err = ::split(Rest, ':', Split))
-          return Err;
-      }
-      break;
-    case 'S': { // Stack natural alignment.
-      uint64_t Alignment;
-      if (Error Err = getIntInBytes(Tok, Alignment))
+  // Aliases used below.
+  StringRef &Tok = Split.first;   // Current token.
+  StringRef &Rest = Split.second; // The rest of the string.
+
+  char SpecifierChar = Tok.front();
+  Tok = Tok.substr(1);
+
+  switch (SpecifierChar) {
+  case 's':
+    // Deprecated, but ignoring here to preserve loading older textual llvm
+    // ASM file
+    break;
+  case 'E':
+    BigEndian = true;
+    break;
+  case 'e':
+    BigEndian = false;
+    break;
+  case 'n': // Native integer types.
+    while (true) {
+      unsigned Width;
+      if (Error Err = getInt(Tok, Width))
         return Err;
-      if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
-        return reportError("Alignment is neither 0 nor a power of 2");
-      StackNaturalAlign = MaybeAlign(Alignment);
-      break;
-    }
-    case 'F': {
-      switch (Tok.front()) {
-      case 'i':
-        TheFunctionPtrAlignType = FunctionPtrAlignType::Independent;
-        break;
-      case 'n':
-        TheFunctionPtrAlignType = FunctionPtrAlignType::MultipleOfFunctionAlign;
+      if (Width == 0)
+        return reportError(
+            "Zero width native integer type in datalayout string");
+      LegalIntWidths.push_back(Width);
+      if (Rest.empty())
         break;
-      default:
-        return reportError("Unknown function pointer alignment type in "
-                           "datalayout string");
-      }
-      Tok = Tok.substr(1);
-      uint64_t Alignment;
-      if (Error Err = getIntInBytes(Tok, Alignment))
+      if (Error Err = ::split(Rest, ':', Split))
         return Err;
-      if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
-        return reportError("Alignment is neither 0 nor a power of 2");
-      FunctionPtrAlign = MaybeAlign(Alignment);
-      break;
     }
-    case 'P': { // Function address space.
-      if (Error Err = getAddrSpace(Tok, ProgramAddrSpace))
-        return Err;
+    break;
+  case 'S': { // Stack natural alignment.
+    uint64_t Alignment;
+    if (Error Err = getIntInBytes(Tok, Alignment))
+      return Err;
+    if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
+      return reportError("Alignment is neither 0 nor a power of 2");
+    StackNaturalAlign = MaybeAlign(Alignment);
+    break;
+  }
+  case 'F': {
+    switch (Tok.front()) {
+    case 'i':
+      TheFunctionPtrAlignType = FunctionPtrAlignType::Independent;
       break;
-    }
-    case 'A': { // Default stack/alloca address space.
-      if (Error Err = getAddrSpace(Tok, AllocaAddrSpace))
-        return Err;
+    case 'n':
+      TheFunctionPtrAlignType = FunctionPtrAlignType::MultipleOfFunctionAlign;
       break;
+    default:
+      return reportError("Unknown function pointer alignment type in "
+                         "datalayout string");
     }
-    case 'G': { // Default address space for global variables.
-      if (Error Err = getAddrSpace(Tok, DefaultGlobalsAddrSpace))
-        return Err;
+    Tok = Tok.substr(1);
+    uint64_t Alignment;
+    if (Error Err = getIntInBytes(Tok, Alignment))
+      return Err;
+    if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
+      return reportError("Alignment is neither 0 nor a power of 2");
+    FunctionPtrAlign = MaybeAlign(Alignment);
+    break;
+  }
+  case 'P': { // Function address space.
+    if (Error Err = getAddrSpace(Tok, ProgramAddrSpace))
+      return Err;
+    break;
+  }
+  case 'A': { // Default stack/alloca address space.
+    if (Error Err = getAddrSpace(Tok, AllocaAddrSpace))
+      return Err;
+    break;
+  }
+  case 'G': { // Default address space for global variables.
+    if (Error Err = getAddrSpace(Tok, DefaultGlobalsAddrSpace))
+      return Err;
+    break;
+  }
+  case 'm':
+    if (!Tok.empty())
+      return reportError("Unexpected trailing characters after mangling "
+                         "specifier in datalayout string");
+    if (Rest.empty())
+      return reportError("Expected mangling specifier in datalayout string");
+    if (Rest.size() > 1)
+      return reportError("Unknown mangling specifier in datalayout string");
+    switch (Rest[0]) {
+    default:
+      return reportError("Unknown mangling in datalayout string");
+    case 'e':
+      ManglingMode = MM_ELF;
+      break;
+    case 'l':
+      ManglingMode = MM_GOFF;
+      break;
+    case 'o':
+      ManglingMode = MM_MachO;
       break;
-    }
     case 'm':
-      if (!Tok.empty())
-        return reportError("Unexpected trailing characters after mangling "
-                           "specifier in datalayout string");
-      if (Rest.empty())
-        return reportError("Expected mangling specifier in datalayout string");
-      if (Rest.size() > 1)
-        return reportError("Unknown mangling specifier in datalayout string");
-      switch(Rest[0]) {
-      default:
-        return reportError("Unknown mangling in datalayout string");
-      case 'e':
-        ManglingMode = MM_ELF;
-        break;
-      case 'l':
-        ManglingMode = MM_GOFF;
-        break;
-      case 'o':
-        ManglingMode = MM_MachO;
-        break;
-      case 'm':
-        ManglingMode = MM_Mips;
-        break;
-      case 'w':
-        ManglingMode = MM_WinCOFF;
-        break;
-      case 'x':
-        ManglingMode = MM_WinCOFFX86;
-        break;
-      case 'a':
-        ManglingMode = MM_XCOFF;
-        break;
-      }
+      ManglingMode = MM_Mips;
       break;
-    default:
-      return reportError("Unknown specifier in datalayout string");
+    case 'w':
+      ManglingMode = MM_WinCOFF;
+      break;
+    case 'x':
+      ManglingMode = MM_WinCOFFX86;
+      break;
+    case 'a':
+      ManglingMode = MM_XCOFF;
       break;
     }
+    break;
+  default:
+    return reportError("Unknown specifier in datalayout string");
   }
 
   return Error::success();
 }
 
-static SmallVectorImpl<DataLayout::PrimitiveSpec>::const_iterator
-findPrimitiveSpecLowerBound(
-    const SmallVectorImpl<DataLayout::PrimitiveSpec> &Specs,
-    uint32_t BitWidth) {
-  return partition_point(Specs, [BitWidth](const DataLayout::PrimitiveSpec &E) {
-    return E.BitWidth < BitWidth;
-  });
-}
-
-Error DataLayout::setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
-                                   Align ABIAlign, Align PrefAlign) {
-  // AlignmentsTy::ABIAlign and AlignmentsTy::PrefAlign were once stored as
-  // uint16_t, it is unclear if there are requirements for alignment to be less
-  // than 2^16 other than storage. In the meantime we leave the restriction as
-  // an assert. See D67400 for context.
-  assert(Log2(ABIAlign) < 16 && Log2(PrefAlign) < 16 && "Alignment too big");
-  if (!isUInt<24>(BitWidth))
-    return reportError("Invalid bit width, must be a 24-bit integer");
-  if (PrefAlign < ABIAlign)
-    return reportError(
-        "Preferred alignment cannot be less than the ABI alignment");
+Error DataLayout::parseLayoutString(StringRef LayoutString) {
+  StringRepresentation = std::string(LayoutString);
 
+  if (LayoutString.empty())
+    return Error::success();
+
+  // Split the data layout string into specifications separated by '-' and
+  // parse each specification individually, updating internal data structures.
+  for (StringRef Spec : split(LayoutString, '-')) {
+    if (Spec.empty())
+      return createStringError("empty specification is not allowed");
+    if (Error Err = parseSpecification(Spec))
+      return Err;
+  }
+
+  return Error::success();
+}
+
+void DataLayout::setPrimitiveSpec(char Specifier, uint32_t BitWidth,
+                                  Align ABIAlign, Align PrefAlign) {
   SmallVectorImpl<PrimitiveSpec> *Specs;
   switch (Specifier) {
-  case TypeSpecifier::Aggregate:
-    StructABIAlignment = ABIAlign;
-    StructPrefAlignment = PrefAlign;
-    return Error::success();
-  case TypeSpecifier::Integer:
+  default:
+    llvm_unreachable("Unexpected specifier");
+  case 'i':
     Specs = &IntSpecs;
     break;
-  case TypeSpecifier::Float:
+  case 'f':
     Specs = &FloatSpecs;
     break;
-  case TypeSpecifier::Vector:
+  case 'v':
     Specs = &VectorSpecs;
     break;
   }
 
-  auto I = partition_point(*Specs, [BitWidth](const PrimitiveSpec &E) {
-    return E.BitWidth < BitWidth;
-  });
+  auto I = lower_bound(*Specs, BitWidth, LessPrimitiveBitWidth());
   if (I != Specs->end() && I->BitWidth == BitWidth) {
     // Update the abi, preferred alignments.
     I->ABIAlign = ABIAlign;
@@ -631,16 +704,12 @@ Error DataLayout::setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
     // Insert before I to keep the vector sorted.
     Specs->insert(I, PrimitiveSpec{BitWidth, ABIAlign, PrefAlign});
   }
-  return Error::success();
 }
 
 const DataLayout::PointerSpec &
 DataLayout::getPointerSpec(uint32_t AddrSpace) const {
   if (AddrSpace != 0) {
-    auto I = lower_bound(PointerSpecs, AddrSpace,
-                         [](const PointerSpec &Spec, uint32_t AddrSpace) {
-                           return Spec.AddrSpace < AddrSpace;
-                         });
+    auto I = lower_bound(PointerSpecs, AddrSpace, LessPointerAddrSpace());
     if (I != PointerSpecs.end() && I->AddrSpace == AddrSpace)
       return *I;
   }
@@ -649,19 +718,10 @@ DataLayout::getPointerSpec(uint32_t AddrSpace) const {
   return PointerSpecs[0];
 }
 
-Error DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
-                                 Align ABIAlign, Align PrefAlign,
-                                 uint32_t IndexBitWidth) {
-  if (PrefAlign < ABIAlign)
-    return reportError(
-        "Preferred alignment cannot be less than the ABI alignment");
-  if (IndexBitWidth > BitWidth)
-    return reportError("Index width cannot be larger than pointer width");
-
-  auto I = lower_bound(PointerSpecs, AddrSpace,
-                       [](const PointerSpec &A, uint32_t AddrSpace) {
-                         return A.AddrSpace < AddrSpace;
-                       });
+void DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
+                                Align ABIAlign, Align PrefAlign,
+                                uint32_t IndexBitWidth) {
+  auto I = lower_bound(PointerSpecs, AddrSpace, LessPointerAddrSpace());
   if (I == PointerSpecs.end() || I->AddrSpace != AddrSpace) {
     PointerSpecs.insert(I, PointerSpec{AddrSpace, BitWidth, ABIAlign, PrefAlign,
                                        IndexBitWidth});
@@ -671,12 +731,11 @@ Error DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
     I->PrefAlign = PrefAlign;
     I->IndexBitWidth = IndexBitWidth;
   }
-  return Error::success();
 }
 
 Align DataLayout::getIntegerAlignment(uint32_t BitWidth,
                                       bool abi_or_pref) const {
-  auto I = findPrimitiveSpecLowerBound(IntSpecs, BitWidth);
+  auto I = lower_bound(IntSpecs, BitWidth, LessPrimitiveBitWidth());
   // If we don't have an exact match, use alignment of next larger integer
   // type. If there is none, use alignment of largest integer type by going
   // back one element.
@@ -792,7 +851,7 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   case Type::FP128TyID:
   case Type::X86_FP80TyID: {
     unsigned BitWidth = getTypeSizeInBits(Ty).getFixedValue();
-    auto I = findPrimitiveSpecLowerBound(FloatSpecs, BitWidth);
+    auto I = lower_bound(FloatSpecs, BitWidth, LessPrimitiveBitWidth());
     if (I != FloatSpecs.end() && I->BitWidth == BitWidth)
       return abi_or_pref ? I->ABIAlign : I->PrefAlign;
 
@@ -807,7 +866,7 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   case Type::FixedVectorTyID:
   case Type::ScalableVectorTyID: {
     unsigned BitWidth = getTypeSizeInBits(Ty).getKnownMinValue();
-    auto I = findPrimitiveSpecLowerBound(VectorSpecs, BitWidth);
+    auto I = lower_bound(VectorSpecs, BitWidth, LessPrimitiveBitWidth());
     if (I != VectorSpecs.end() && I->BitWidth == BitWidth)
       return abi_or_pref ? I->ABIAlign : I->PrefAlign;
 
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 7a8a6c8daf95938..35d0a2eb52dfc76 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1443,22 +1443,17 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
   bool RelocateWithSymbol =
       shouldRelocateWithSymbol(Asm, Target, SymA, C, Type) ||
       (Parent->getType() == ELF::SHT_LLVM_CALL_GRAPH_PROFILE);
-  uint64_t Addend = 0;
-
-  FixedValue = !RelocateWithSymbol && SymA && !SymA->isUndefined()
-                   ? C + Asm.getSymbolOffset(*SymA)
-                   : C;
-  if (usesRela(TO, FixupSection)) {
-    Addend = FixedValue;
-    FixedValue = 0;
-  }
+  uint64_t Addend = !RelocateWithSymbol && SymA && !SymA->isUndefined()
+                        ? C + Asm.getSymbolOffset(*SymA)
+                        : C;
+  FixedValue = usesRela(TO, FixupSection) ? 0 : Addend;
 
   if (!RelocateWithSymbol) {
     const auto *SectionSymbol =
         SecA ? cast<MCSymbolELF>(SecA->getBeginSymbol()) : nullptr;
     if (SectionSymbol)
       SectionSymbol->setUsedInReloc();
-    ELFRelocationEntry Rec(FixupOffset, SectionSymbol, Type, Addend, SymA, C);
+    ELFRelocationEntry Rec(FixupOffset, SectionSymbol, Type, Addend, SymA);
     Relocations[&FixupSection].push_back(Rec);
     return;
   }
@@ -1473,7 +1468,7 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
     else
       RenamedSymA->setUsedInReloc();
   }
-  ELFRelocationEntry Rec(FixupOffset, RenamedSymA, Type, Addend, SymA, C);
+  ELFRelocationEntry Rec(FixupOffset, RenamedSymA, Type, Addend, SymA);
   Relocations[&FixupSection].push_back(Rec);
 }
 
diff --git a/llvm/lib/MC/MCAsmInfoDarwin.cpp b/llvm/lib/MC/MCAsmInfoDarwin.cpp
index a10be51ee61ac07..9cba775e9168b6c 100644
--- a/llvm/lib/MC/MCAsmInfoDarwin.cpp
+++ b/llvm/lib/MC/MCAsmInfoDarwin.cpp
@@ -69,15 +69,9 @@ MCAsmInfoDarwin::MCAsmInfoDarwin() {
   InlineAsmEnd = " InlineAsm End";
 
   // Directives:
-  HasWeakDefDirective = true;
   HasWeakDefCanBeHiddenDirective = true;
   WeakRefDirective = "\t.weak_reference ";
   ZeroDirective = "\t.space\t";  // ".space N" emits N zeros.
-  HasMachoZeroFillDirective = true;  // Uses .zerofill
-  HasMachoTBSSDirective = true; // Uses .tbss
-
-  // FIXME: Change this once MC is the system assembler.
-  HasAggressiveSymbolFolding = false;
 
   HiddenVisibilityAttr = MCSA_PrivateExtern;
   HiddenDeclarationVisibilityAttr = MCSA_Invalid;
@@ -87,7 +81,6 @@ MCAsmInfoDarwin::MCAsmInfoDarwin() {
 
   HasDotTypeDotSizeDirective = false;
   HasNoDeadStrip = true;
-  HasAltEntry = true;
 
   DwarfUsesRelocationsAcrossSections = false;
   SetDirectiveSuppressesReloc = true;
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index cbeb41f58c99be7..55139dae7b1d71f 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -167,18 +167,12 @@ bool MCAssembler::evaluateFixup(const MCFixup &Fixup, const MCFragment *DF,
     }
   }
 
-  assert(getBackendPtr() && "Expected assembler backend");
-  bool IsTarget = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags &
-                  MCFixupKindInfo::FKF_IsTarget;
-
-  if (IsTarget)
+  unsigned FixupFlags = getBackend().getFixupKindInfo(Fixup.getKind()).Flags;
+  if (FixupFlags & MCFixupKindInfo::FKF_IsTarget)
     return getBackend().evaluateTargetFixup(*this, Fixup, DF, Target, STI,
                                             Value, WasForced);
 
-  unsigned FixupFlags = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags;
-  bool IsPCRel = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags &
-                 MCFixupKindInfo::FKF_IsPCRel;
-
+  bool IsPCRel = FixupFlags & MCFixupKindInfo::FKF_IsPCRel;
   bool IsResolved = false;
   if (IsPCRel) {
     if (Target.getSymB()) {
@@ -213,8 +207,7 @@ bool MCAssembler::evaluateFixup(const MCFixup &Fixup, const MCFragment *DF,
       Value -= getSymbolOffset(Sym);
   }
 
-  bool ShouldAlignPC = getBackend().getFixupKindInfo(Fixup.getKind()).Flags &
-                       MCFixupKindInfo::FKF_IsAlignedDownTo32Bits;
+  bool ShouldAlignPC = FixupFlags & MCFixupKindInfo::FKF_IsAlignedDownTo32Bits;
   assert((ShouldAlignPC ? IsPCRel : true) &&
     "FKF_IsAlignedDownTo32Bits is only allowed on PC-relative fixups!");
 
diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp
index 1297dc3828b581b..2db59f8fc944264 100644
--- a/llvm/lib/MC/MCDwarf.cpp
+++ b/llvm/lib/MC/MCDwarf.cpp
@@ -320,9 +320,10 @@ MCDwarfLineTableHeader::Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params,
 static const MCExpr *forceExpAbs(MCStreamer &OS, const MCExpr* Expr) {
   MCContext &Context = OS.getContext();
   assert(!isa<MCSymbolRefExpr>(Expr));
-  if (Context.getAsmInfo()->hasAggressiveSymbolFolding())
+  if (!Context.getAsmInfo()->doesSetDirectiveSuppressReloc())
     return Expr;
 
+  // On Mach-O, try to avoid a relocation by using a set directive.
   MCSymbol *ABS = Context.createTempSymbol();
   OS.emitAssignment(ABS, Expr);
   return MCSymbolRefExpr::create(ABS, Context);
diff --git a/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp b/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
index 5bba1dea9adf0e3..cf3d884bee3bd8e 100644
--- a/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
+++ b/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
@@ -22,7 +22,7 @@ using namespace object;
 using SectionPred = std::function<bool(const Section &Sec)>;
 
 static bool isDebugSection(const Section &Sec) {
-  return Sec.Name.starts_with(".debug");
+  return Sec.Name.starts_with(".debug") || Sec.Name.starts_with("reloc..debug");
 }
 
 static bool isLinkerSection(const Section &Sec) {
diff --git a/llvm/lib/ObjCopy/wasm/WasmObject.cpp b/llvm/lib/ObjCopy/wasm/WasmObject.cpp
index 28a2de6e6e4f124..0b543b793f049ed 100644
--- a/llvm/lib/ObjCopy/wasm/WasmObject.cpp
+++ b/llvm/lib/ObjCopy/wasm/WasmObject.cpp
@@ -25,8 +25,22 @@ void Object::addSectionWithOwnedContents(
 }
 
 void Object::removeSections(function_ref<bool(const Section &)> ToRemove) {
-  // TODO: remove reloc sections for the removed section, handle symbols, etc.
-  llvm::erase_if(Sections, ToRemove);
+  if (isRelocatableObject) {
+    // For relocatable objects, avoid actually removing any sections,
+    // since that can invalidate the symbol table and relocation sections.
+    // TODO: Allow removal of sections by re-generating symbol table and
+    // relocation sections here instead.
+    for (auto &Sec : Sections) {
+      if (ToRemove(Sec)) {
+        Sec.Name = ".objcopy.removed";
+        Sec.SectionType = wasm::WASM_SEC_CUSTOM;
+        Sec.Contents = {};
+        Sec.HeaderSecSizeEncodingLen = std::nullopt;
+      }
+    }
+  } else {
+    llvm::erase_if(Sections, ToRemove);
+  }
 }
 
 } // end namespace wasm
diff --git a/llvm/lib/ObjCopy/wasm/WasmObject.h b/llvm/lib/ObjCopy/wasm/WasmObject.h
index f860ec697e5684d..374e523cd879b65 100644
--- a/llvm/lib/ObjCopy/wasm/WasmObject.h
+++ b/llvm/lib/ObjCopy/wasm/WasmObject.h
@@ -32,6 +32,7 @@ struct Object {
   llvm::wasm::WasmObjectHeader Header;
   // For now don't discriminate between kinds of sections.
   std::vector<Section> Sections;
+  bool isRelocatableObject = false;
 
   void addSectionWithOwnedContents(Section NewSection,
                                    std::unique_ptr<MemoryBuffer> &&Content);
diff --git a/llvm/lib/ObjCopy/wasm/WasmReader.cpp b/llvm/lib/ObjCopy/wasm/WasmReader.cpp
index 578e78955af3e45..420d17f9864323f 100644
--- a/llvm/lib/ObjCopy/wasm/WasmReader.cpp
+++ b/llvm/lib/ObjCopy/wasm/WasmReader.cpp
@@ -18,6 +18,7 @@ using namespace llvm::wasm;
 Expected<std::unique_ptr<Object>> Reader::create() const {
   auto Obj = std::make_unique<Object>();
   Obj->Header = WasmObj.getHeader();
+  Obj->isRelocatableObject = WasmObj.isRelocatableObject();
   std::vector<Section> Sections;
   Obj->Sections.reserve(WasmObj.getNumSections());
   for (const SectionRef &Sec : WasmObj.sections()) {
diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp
index e67b02405a3a177..cc0a5da7e0d16a8 100644
--- a/llvm/lib/Object/COFFImportFile.cpp
+++ b/llvm/lib/Object/COFFImportFile.cpp
@@ -705,7 +705,7 @@ Error writeImportLibrary(StringRef ImportName, StringRef Path,
         Name = std::string(SymbolName);
       } else {
         Expected<std::string> ReplacedName =
-            replace(SymbolName, E.Name, E.ExtName);
+            object::replace(SymbolName, E.Name, E.ExtName);
         if (!ReplacedName)
           return ReplacedName.takeError();
         Name.swap(*ReplacedName);
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index ff55a847c34327d..e2d1723b1fa9813 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -2341,7 +2341,7 @@ void MachOObjectFile::getRelocationTypeName(
         "ARM64_RELOC_PAGEOFF12",          "ARM64_RELOC_GOT_LOAD_PAGE21",
         "ARM64_RELOC_GOT_LOAD_PAGEOFF12", "ARM64_RELOC_POINTER_TO_GOT",
         "ARM64_RELOC_TLVP_LOAD_PAGE21",   "ARM64_RELOC_TLVP_LOAD_PAGEOFF12",
-        "ARM64_RELOC_ADDEND"
+        "ARM64_RELOC_ADDEND",             "ARM64_RELOC_AUTHENTICATED_POINTER"
       };
 
       if (RType >= std::size(Table))
diff --git a/llvm/lib/Object/RelocationResolver.cpp b/llvm/lib/Object/RelocationResolver.cpp
index 7f90834c3d21997..d9bb8f175ef83ac 100644
--- a/llvm/lib/Object/RelocationResolver.cpp
+++ b/llvm/lib/Object/RelocationResolver.cpp
@@ -47,7 +47,6 @@ static bool supportsX86_64(uint64_t Type) {
   case ELF::R_X86_64_PC64:
   case ELF::R_X86_64_32:
   case ELF::R_X86_64_32S:
-  case ELF::R_X86_64_GLOB_DAT:
     return true;
   default:
     return false;
@@ -69,8 +68,6 @@ static uint64_t resolveX86_64(uint64_t Type, uint64_t Offset, uint64_t S,
   case ELF::R_X86_64_32:
   case ELF::R_X86_64_32S:
     return (S + Addend) & 0xFFFFFFFF;
-  case ELF::R_X86_64_GLOB_DAT:
-    return S;
   default:
     llvm_unreachable("Invalid relocation type");
   }
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 1a192f72ae4d606..89ffc383a4a6ece 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -547,6 +547,20 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCase(EF_RISCV_RVE);
     BCase(EF_RISCV_TSO);
     break;
+  case ELF::EM_SPARC32PLUS:
+    BCase(EF_SPARC_32PLUS);
+    BCase(EF_SPARC_SUN_US1);
+    BCase(EF_SPARC_SUN_US3);
+    BCase(EF_SPARC_HAL_R1);
+    break;
+  case ELF::EM_SPARCV9:
+    BCase(EF_SPARC_SUN_US1);
+    BCase(EF_SPARC_SUN_US3);
+    BCase(EF_SPARC_HAL_R1);
+    BCaseMask(EF_SPARCV9_RMO, EF_SPARCV9_MM);
+    BCaseMask(EF_SPARCV9_PSO, EF_SPARCV9_MM);
+    BCaseMask(EF_SPARCV9_TSO, EF_SPARCV9_MM);
+    break;
   case ELF::EM_XTENSA:
     BCase(EF_XTENSA_XT_INSN);
     BCaseMask(EF_XTENSA_MACH_NONE, EF_XTENSA_MACH);
diff --git a/llvm/lib/ObjectYAML/WasmEmitter.cpp b/llvm/lib/ObjectYAML/WasmEmitter.cpp
index 9b8fd11f85437e9..4f384928b89f1df 100644
--- a/llvm/lib/ObjectYAML/WasmEmitter.cpp
+++ b/llvm/lib/ObjectYAML/WasmEmitter.cpp
@@ -604,7 +604,8 @@ bool WasmWriter::writeWasm(raw_ostream &OS) {
     if (auto S = dyn_cast<WasmYAML::CustomSection>(Sec.get()))
       SecName = S->Name;
     if (!Checker.isValidSectionOrder(Sec->Type, SecName)) {
-      reportError("out of order section type: " + Twine(Sec->Type));
+      reportError("out of order section type: " +
+                  wasm::sectionTypeToString(Sec->Type));
       return false;
     }
     encodeULEB128(Sec->Type, OS);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index c528863d1870a4a..17eed97fd950c98 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -249,6 +249,7 @@
 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
 #include "llvm/Transforms/Scalar/LoopSink.h"
 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
+#include "llvm/Transforms/Scalar/LoopTermFold.h"
 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
 #include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
@@ -334,8 +335,6 @@ cl::opt<bool> PrintPipelinePasses(
              "(best-effort only)."));
 } // namespace llvm
 
-extern cl::opt<std::string> UseCtxProfile;
-
 AnalysisKey NoOpModuleAnalysis::Key;
 AnalysisKey NoOpCGSCCAnalysis::Key;
 AnalysisKey NoOpFunctionAnalysis::Key;
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index 4e12dd2226c4d22..933fe89e53a94cf 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -48,15 +48,12 @@ static TargetMachine *unwrap(LLVMTargetMachineRef P) {
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLVMPassBuilderOptions,
                                    LLVMPassBuilderOptionsRef)
 
-LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
-                           LLVMTargetMachineRef TM,
-                           LLVMPassBuilderOptionsRef Options) {
-  TargetMachine *Machine = unwrap(TM);
-  LLVMPassBuilderOptions *PassOpts = unwrap(Options);
+static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
+                              TargetMachine *Machine,
+                              LLVMPassBuilderOptions *PassOpts) {
   bool Debug = PassOpts->DebugLogging;
   bool VerifyEach = PassOpts->VerifyEach;
 
-  Module *Mod = unwrap(M);
   PassInstrumentationCallbacks PIC;
   PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC);
 
@@ -80,18 +77,45 @@ LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
   SI.registerCallbacks(PIC, &MAM);
-  ModulePassManager MPM;
-  if (VerifyEach) {
-    MPM.addPass(VerifierPass());
-  }
-  if (auto Err = PB.parsePassPipeline(MPM, Passes)) {
-    return wrap(std::move(Err));
+
+  // Run the pipeline.
+  if (Fun) {
+    FunctionPassManager FPM;
+    if (VerifyEach)
+      FPM.addPass(VerifierPass());
+    if (auto Err = PB.parsePassPipeline(FPM, Passes))
+      return wrap(std::move(Err));
+    FPM.run(*Fun, FAM);
+  } else {
+    ModulePassManager MPM;
+    if (VerifyEach)
+      MPM.addPass(VerifierPass());
+    if (auto Err = PB.parsePassPipeline(MPM, Passes))
+      return wrap(std::move(Err));
+    MPM.run(*Mod, MAM);
   }
 
-  MPM.run(*Mod, MAM);
   return LLVMErrorSuccess;
 }
 
+LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
+                           LLVMTargetMachineRef TM,
+                           LLVMPassBuilderOptionsRef Options) {
+  TargetMachine *Machine = unwrap(TM);
+  LLVMPassBuilderOptions *PassOpts = unwrap(Options);
+  Module *Mod = unwrap(M);
+  return runPasses(Mod, nullptr, Passes, Machine, PassOpts);
+}
+
+LLVMErrorRef LLVMRunPassesOnFunction(LLVMValueRef F, const char *Passes,
+                                     LLVMTargetMachineRef TM,
+                                     LLVMPassBuilderOptionsRef Options) {
+  TargetMachine *Machine = unwrap(TM);
+  LLVMPassBuilderOptions *PassOpts = unwrap(Options);
+  Function *Fun = unwrap<Function>(F);
+  return runPasses(Fun->getParent(), Fun, Passes, Machine, PassOpts);
+}
+
 LLVMPassBuilderOptionsRef LLVMCreatePassBuilderOptions() {
   return wrap(new LLVMPassBuilderOptions());
 }
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 0201e69f3e216a4..1184123c7710f05 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -844,7 +844,8 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
   }
 
   // Perform PGO instrumentation.
-  MPM.addPass(PGOInstrumentationGen(IsCS));
+  MPM.addPass(PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
+                                         : PGOInstrumentationType::FDO));
 
   addPostPGOLoopRotation(MPM, Level);
   // Add the profile lowering pass.
@@ -879,7 +880,8 @@ void PassBuilder::addPGOInstrPassesForO0(
   }
 
   // Perform PGO instrumentation.
-  MPM.addPass(PGOInstrumentationGen(IsCS));
+  MPM.addPass(PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
+                                         : PGOInstrumentationType::FDO));
   // Add the profile lowering pass.
   InstrProfOptions Options;
   if (!ProfileFile.empty())
@@ -1193,7 +1195,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
                       PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile,
                       PGOOpt->FS);
   } else if (IsCtxProfGen || IsCtxProfUse) {
-    MPM.addPass(PGOInstrumentationGen(false));
+    MPM.addPass(PGOInstrumentationGen(PGOInstrumentationType::CTXPROF));
     // In pre-link, we just want the instrumented IR. We use the contextual
     // profile in the post-thinlink phase.
     // The instrumentation will be removed in post-thinlink after IPO.
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 18f4aa19224da05..6b5e1cf83c4698b 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -20,7 +20,7 @@
 #endif
 MODULE_ANALYSIS("callgraph", CallGraphAnalysis())
 MODULE_ANALYSIS("collector-metadata", CollectorMetadataAnalysis())
-MODULE_ANALYSIS("ctx-prof-analysis", CtxProfAnalysis(UseCtxProfile))
+MODULE_ANALYSIS("ctx-prof-analysis", CtxProfAnalysis())
 MODULE_ANALYSIS("dxil-metadata", DXILMetadataAnalysis())
 MODULE_ANALYSIS("dxil-resource", DXILResourceAnalysis())
 MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis())
@@ -56,6 +56,8 @@ MODULE_PASS("constmerge", ConstantMergePass())
 MODULE_PASS("coro-cleanup", CoroCleanupPass())
 MODULE_PASS("coro-early", CoroEarlyPass())
 MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass())
+MODULE_PASS("ctx-instr-gen",
+            PGOInstrumentationGen(PGOInstrumentationType::CTXPROF))
 MODULE_PASS("deadargelim", DeadArgumentEliminationPass())
 MODULE_PASS("debugify", NewPMDebugifyPass())
 MODULE_PASS("dfsan", DataFlowSanitizerPass())
@@ -644,6 +646,7 @@ LOOP_PASS("loop-idiom-vectorize", LoopIdiomVectorizePass())
 LOOP_PASS("loop-instsimplify", LoopInstSimplifyPass())
 LOOP_PASS("loop-predication", LoopPredicationPass())
 LOOP_PASS("loop-reduce", LoopStrengthReducePass())
+LOOP_PASS("loop-term-fold", LoopTermFoldPass())
 LOOP_PASS("loop-simplifycfg", LoopSimplifyCFGPass())
 LOOP_PASS("loop-unroll-full", LoopFullUnrollPass())
 LOOP_PASS("loop-versioning-licm", LoopVersioningLICMPass())
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index 21ce0ac17d61860..18643c6b44485ea 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -1533,9 +1533,9 @@ LineCoverageStats::LineCoverageStats(
 
   // if there is any starting segment at this line with a counter, it must be
   // mapped
-  Mapped |= std::any_of(
-      LineSegments.begin(), LineSegments.end(),
-      [](const auto *Seq) { return Seq->IsRegionEntry && Seq->HasCount; });
+  Mapped |= any_of(LineSegments, [](const auto *Seq) {
+    return Seq->IsRegionEntry && Seq->HasCount;
+  });
 
   if (!Mapped) {
     return;
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 3c7dbb70c9f4b7a..c243df7fc864ee5 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -1031,7 +1031,10 @@ void CallBrInst::setDefaultDest(BasicBlock *BB) {
   cast<llvm::CallBrInst>(Val)->setDefaultDest(cast<llvm::BasicBlock>(BB->Val));
 }
 void CallBrInst::setIndirectDest(unsigned Idx, BasicBlock *BB) {
-  Ctx.getTracker().emplaceIfTracking<CallBrInstSetIndirectDest>(this, Idx);
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&CallBrInst::getIndirectDest,
+                                              &CallBrInst::setIndirectDest>>(
+          this, Idx);
   cast<llvm::CallBrInst>(Val)->setIndirectDest(Idx,
                                                cast<llvm::BasicBlock>(BB->Val));
 }
@@ -1102,10 +1105,10 @@ Value *PHINode::getIncomingValue(unsigned Idx) const {
   return Ctx.getValue(cast<llvm::PHINode>(Val)->getIncomingValue(Idx));
 }
 void PHINode::setIncomingValue(unsigned Idx, Value *V) {
-  auto &Tracker = Ctx.getTracker();
-  Tracker.emplaceIfTracking<PHISetIncoming>(this, Idx,
-                                            PHISetIncoming::What::Value);
-
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&PHINode::getIncomingValue,
+                                              &PHINode::setIncomingValue>>(this,
+                                                                           Idx);
   cast<llvm::PHINode>(Val)->setIncomingValue(Idx, V->Val);
 }
 BasicBlock *PHINode::getIncomingBlock(unsigned Idx) const {
@@ -1118,9 +1121,13 @@ BasicBlock *PHINode::getIncomingBlock(const Use &U) const {
   return cast<BasicBlock>(Ctx.getValue(BB));
 }
 void PHINode::setIncomingBlock(unsigned Idx, BasicBlock *BB) {
-  auto &Tracker = Ctx.getTracker();
-  Tracker.emplaceIfTracking<PHISetIncoming>(this, Idx,
-                                            PHISetIncoming::What::Block);
+  // Helper to disambiguate PHINode::getIncomingBlock(unsigned).
+  constexpr BasicBlock *(PHINode::*GetIncomingBlockFn)(unsigned) const =
+      &PHINode::getIncomingBlock;
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetterWithIdx<GetIncomingBlockFn, &PHINode::setIncomingBlock>>(
+          this, Idx);
   cast<llvm::PHINode>(Val)->setIncomingBlock(Idx,
                                              cast<llvm::BasicBlock>(BB->Val));
 }
@@ -1219,6 +1226,318 @@ static llvm::Instruction::CastOps getLLVMCastOp(Instruction::Opcode Opc) {
   }
 }
 
+/// \Returns the LLVM opcode that corresponds to \p Opc.
+static llvm::Instruction::UnaryOps getLLVMUnaryOp(Instruction::Opcode Opc) {
+  switch (Opc) {
+  case Instruction::Opcode::FNeg:
+    return static_cast<llvm::Instruction::UnaryOps>(llvm::Instruction::FNeg);
+  default:
+    llvm_unreachable("Not a unary op!");
+  }
+}
+
+SwitchInst *SwitchInst::create(Value *V, BasicBlock *Dest, unsigned NumCases,
+                               BasicBlock::iterator WhereIt,
+                               BasicBlock *WhereBB, Context &Ctx,
+                               const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::SwitchInst *LLVMSwitch =
+      Builder.CreateSwitch(V->Val, cast<llvm::BasicBlock>(Dest->Val), NumCases);
+  return Ctx.createSwitchInst(LLVMSwitch);
+}
+
+Value *SwitchInst::getCondition() const {
+  return Ctx.getValue(cast<llvm::SwitchInst>(Val)->getCondition());
+}
+
+void SwitchInst::setCondition(Value *V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&SwitchInst::getCondition, &SwitchInst::setCondition>>(
+          this);
+  cast<llvm::SwitchInst>(Val)->setCondition(V->Val);
+}
+
+BasicBlock *SwitchInst::getDefaultDest() const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::SwitchInst>(Val)->getDefaultDest()));
+}
+
+void SwitchInst::setDefaultDest(BasicBlock *DefaultCase) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&SwitchInst::getDefaultDest,
+                                       &SwitchInst::setDefaultDest>>(this);
+  cast<llvm::SwitchInst>(Val)->setDefaultDest(
+      cast<llvm::BasicBlock>(DefaultCase->Val));
+}
+ConstantInt *SwitchInst::findCaseDest(BasicBlock *BB) {
+  auto *LLVMC = cast<llvm::SwitchInst>(Val)->findCaseDest(
+      cast<llvm::BasicBlock>(BB->Val));
+  return LLVMC != nullptr ? cast<ConstantInt>(Ctx.getValue(LLVMC)) : nullptr;
+}
+
+void SwitchInst::addCase(ConstantInt *OnVal, BasicBlock *Dest) {
+  Ctx.getTracker().emplaceIfTracking<SwitchAddCase>(this, OnVal);
+  // TODO: Track this!
+  cast<llvm::SwitchInst>(Val)->addCase(cast<llvm::ConstantInt>(OnVal->Val),
+                                       cast<llvm::BasicBlock>(Dest->Val));
+}
+
+SwitchInst::CaseIt SwitchInst::removeCase(CaseIt It) {
+  auto &Case = *It;
+  Ctx.getTracker().emplaceIfTracking<SwitchRemoveCase>(
+      this, Case.getCaseValue(), Case.getCaseSuccessor());
+
+  auto *LLVMSwitch = cast<llvm::SwitchInst>(Val);
+  unsigned CaseNum = It - case_begin();
+  llvm::SwitchInst::CaseIt LLVMIt(LLVMSwitch, CaseNum);
+  auto LLVMCaseIt = LLVMSwitch->removeCase(LLVMIt);
+  unsigned Num = LLVMCaseIt - LLVMSwitch->case_begin();
+  return CaseIt(this, Num);
+}
+
+BasicBlock *SwitchInst::getSuccessor(unsigned Idx) const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::SwitchInst>(Val)->getSuccessor(Idx)));
+}
+
+void SwitchInst::setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&SwitchInst::getSuccessor,
+                                              &SwitchInst::setSuccessor>>(this,
+                                                                          Idx);
+  cast<llvm::SwitchInst>(Val)->setSuccessor(
+      Idx, cast<llvm::BasicBlock>(NewSucc->Val));
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             BBIterator WhereIt, BasicBlock *WhereBB,
+                             Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *NewLLVMV = Builder.CreateUnOp(getLLVMUnaryOp(Op), OpV->Val, Name);
+  if (auto *NewUnOpV = dyn_cast<llvm::UnaryOperator>(NewLLVMV)) {
+    return Ctx.createUnaryOperator(NewUnOpV);
+  }
+  assert(isa<llvm::Constant>(NewLLVMV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewLLVMV));
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             Instruction *InsertBefore, Context &Ctx,
+                             const Twine &Name) {
+  return create(Op, OpV, InsertBefore->getIterator(), InsertBefore->getParent(),
+                Ctx, Name);
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             BasicBlock *InsertAfter, Context &Ctx,
+                             const Twine &Name) {
+  return create(Op, OpV, InsertAfter->end(), InsertAfter, Ctx, Name);
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom, BBIterator WhereIt,
+                                            BasicBlock *WhereBB, Context &Ctx,
+                                            const Twine &Name) {
+  auto *NewV = create(Op, OpV, WhereIt, WhereBB, Ctx, Name);
+  if (auto *UnI = dyn_cast<llvm::UnaryOperator>(NewV->Val))
+    UnI->copyIRFlags(CopyFrom->Val);
+  return NewV;
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom,
+                                            Instruction *InsertBefore,
+                                            Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, OpV, CopyFrom, InsertBefore->getIterator(),
+                               InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom,
+                                            BasicBlock *InsertAtEnd,
+                                            Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, OpV, CopyFrom, InsertAtEnd->end(),
+                               InsertAtEnd, Ctx, Name);
+}
+
+/// \Returns the LLVM opcode that corresponds to \p Opc.
+static llvm::Instruction::BinaryOps getLLVMBinaryOp(Instruction::Opcode Opc) {
+  switch (Opc) {
+  case Instruction::Opcode::Add:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Add);
+  case Instruction::Opcode::FAdd:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FAdd);
+  case Instruction::Opcode::Sub:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Sub);
+  case Instruction::Opcode::FSub:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FSub);
+  case Instruction::Opcode::Mul:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Mul);
+  case Instruction::Opcode::FMul:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FMul);
+  case Instruction::Opcode::UDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::UDiv);
+  case Instruction::Opcode::SDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::SDiv);
+  case Instruction::Opcode::FDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FDiv);
+  case Instruction::Opcode::URem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::URem);
+  case Instruction::Opcode::SRem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::SRem);
+  case Instruction::Opcode::FRem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FRem);
+  case Instruction::Opcode::Shl:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Shl);
+  case Instruction::Opcode::LShr:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::LShr);
+  case Instruction::Opcode::AShr:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::AShr);
+  case Instruction::Opcode::And:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::And);
+  case Instruction::Opcode::Or:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Or);
+  case Instruction::Opcode::Xor:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Xor);
+  default:
+    llvm_unreachable("Not a binary op!");
+  }
+}
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              BBIterator WhereIt, BasicBlock *WhereBB,
+                              Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  llvm::Value *NewV =
+      Builder.CreateBinOp(getLLVMBinaryOp(Op), LHS->Val, RHS->Val, Name);
+  if (auto *NewBinOp = dyn_cast<llvm::BinaryOperator>(NewV))
+    return Ctx.createBinaryOperator(NewBinOp);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              Instruction *InsertBefore, Context &Ctx,
+                              const Twine &Name) {
+  return create(Op, LHS, RHS, InsertBefore->getIterator(),
+                InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              BasicBlock *InsertAtEnd, Context &Ctx,
+                              const Twine &Name) {
+  return create(Op, LHS, RHS, InsertAtEnd->end(), InsertAtEnd, Ctx, Name);
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             BBIterator WhereIt,
+                                             BasicBlock *WhereBB, Context &Ctx,
+                                             const Twine &Name) {
+
+  Value *NewV = create(Op, LHS, RHS, WhereIt, WhereBB, Ctx, Name);
+  if (auto *NewBO = dyn_cast<BinaryOperator>(NewV))
+    cast<llvm::BinaryOperator>(NewBO->Val)->copyIRFlags(CopyFrom->Val);
+  return NewV;
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             Instruction *InsertBefore,
+                                             Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, LHS, RHS, CopyFrom,
+                               InsertBefore->getIterator(),
+                               InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             BasicBlock *InsertAtEnd,
+                                             Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, LHS, RHS, CopyFrom, InsertAtEnd->end(),
+                               InsertAtEnd, Ctx, Name);
+}
+
+void AtomicRMWInst::setAlignment(Align Align) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getAlign,
+                                       &AtomicRMWInst::setAlignment>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setAlignment(Align);
+}
+
+void AtomicRMWInst::setVolatile(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::isVolatile,
+                                       &AtomicRMWInst::setVolatile>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setVolatile(V);
+}
+
+void AtomicRMWInst::setOrdering(AtomicOrdering Ordering) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getOrdering,
+                                       &AtomicRMWInst::setOrdering>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setOrdering(Ordering);
+}
+
+void AtomicRMWInst::setSyncScopeID(SyncScope::ID SSID) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getSyncScopeID,
+                                       &AtomicRMWInst::setSyncScopeID>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setSyncScopeID(SSID);
+}
+
+Value *AtomicRMWInst::getPointerOperand() {
+  return Ctx.getValue(cast<llvm::AtomicRMWInst>(Val)->getPointerOperand());
+}
+
+Value *AtomicRMWInst::getValOperand() {
+  return Ctx.getValue(cast<llvm::AtomicRMWInst>(Val)->getValOperand());
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     BBIterator WhereIt, BasicBlock *WhereBB,
+                                     Context &Ctx, SyncScope::ID SSID,
+                                     const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *LLVMAtomicRMW =
+      Builder.CreateAtomicRMW(Op, Ptr->Val, Val->Val, Align, Ordering, SSID);
+  LLVMAtomicRMW->setName(Name);
+  return Ctx.createAtomicRMWInst(LLVMAtomicRMW);
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     Instruction *InsertBefore, Context &Ctx,
+                                     SyncScope::ID SSID, const Twine &Name) {
+  return create(Op, Ptr, Val, Align, Ordering, InsertBefore->getIterator(),
+                InsertBefore->getParent(), Ctx, SSID, Name);
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     BasicBlock *InsertAtEnd, Context &Ctx,
+                                     SyncScope::ID SSID, const Twine &Name) {
+  return create(Op, Ptr, Val, Align, Ordering, InsertAtEnd->end(), InsertAtEnd,
+                Ctx, SSID, Name);
+}
+
 void AtomicCmpXchgInst::setSyncScopeID(SyncScope::ID SSID) {
   Ctx.getTracker()
       .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getSyncScopeID,
@@ -1454,18 +1773,20 @@ Value *ExtractElementInst::create(Value *Vec, Value *Idx,
   return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
 }
 
-Constant *Constant::createInt(Type *Ty, uint64_t V, Context &Ctx,
-                              bool IsSigned) {
-  llvm::Constant *LLVMC = llvm::ConstantInt::get(Ty, V, IsSigned);
-  return Ctx.getOrCreateConstant(LLVMC);
-}
-
 #ifndef NDEBUG
 void Constant::dumpOS(raw_ostream &OS) const {
   dumpCommonPrefix(OS);
   dumpCommonSuffix(OS);
 }
+#endif // NDEBUG
 
+ConstantInt *ConstantInt::get(Type *Ty, uint64_t V, Context &Ctx,
+                              bool IsSigned) {
+  auto *LLVMC = llvm::ConstantInt::get(Ty, V, IsSigned);
+  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
+}
+
+#ifndef NDEBUG
 void Function::dumpNameAndArgs(raw_ostream &OS) const {
   auto *F = cast<llvm::Function>(Val);
   OS << *F->getReturnType() << " @" << F->getName() << "(";
@@ -1547,6 +1868,10 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
     return It->second.get();
 
   if (auto *C = dyn_cast<llvm::Constant>(LLVMV)) {
+    if (auto *CI = dyn_cast<llvm::ConstantInt>(C)) {
+      It->second = std::unique_ptr<ConstantInt>(new ConstantInt(CI, *this));
+      return It->second.get();
+    }
     if (auto *F = dyn_cast<llvm::Function>(LLVMV))
       It->second = std::unique_ptr<Function>(new Function(F, *this));
     else
@@ -1628,6 +1953,47 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
         new GetElementPtrInst(LLVMGEP, *this));
     return It->second.get();
   }
+  case llvm::Instruction::Switch: {
+    auto *LLVMSwitchInst = cast<llvm::SwitchInst>(LLVMV);
+    It->second =
+        std::unique_ptr<SwitchInst>(new SwitchInst(LLVMSwitchInst, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::FNeg: {
+    auto *LLVMUnaryOperator = cast<llvm::UnaryOperator>(LLVMV);
+    It->second = std::unique_ptr<UnaryOperator>(
+        new UnaryOperator(LLVMUnaryOperator, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Add:
+  case llvm::Instruction::FAdd:
+  case llvm::Instruction::Sub:
+  case llvm::Instruction::FSub:
+  case llvm::Instruction::Mul:
+  case llvm::Instruction::FMul:
+  case llvm::Instruction::UDiv:
+  case llvm::Instruction::SDiv:
+  case llvm::Instruction::FDiv:
+  case llvm::Instruction::URem:
+  case llvm::Instruction::SRem:
+  case llvm::Instruction::FRem:
+  case llvm::Instruction::Shl:
+  case llvm::Instruction::LShr:
+  case llvm::Instruction::AShr:
+  case llvm::Instruction::And:
+  case llvm::Instruction::Or:
+  case llvm::Instruction::Xor: {
+    auto *LLVMBinaryOperator = cast<llvm::BinaryOperator>(LLVMV);
+    It->second = std::unique_ptr<BinaryOperator>(
+        new BinaryOperator(LLVMBinaryOperator, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::AtomicRMW: {
+    auto *LLVMAtomicRMW = cast<llvm::AtomicRMWInst>(LLVMV);
+    It->second =
+        std::unique_ptr<AtomicRMWInst>(new AtomicRMWInst(LLVMAtomicRMW, *this));
+    return It->second.get();
+  }
   case llvm::Instruction::AtomicCmpXchg: {
     auto *LLVMAtomicCmpXchg = cast<llvm::AtomicCmpXchgInst>(LLVMV);
     It->second = std::unique_ptr<AtomicCmpXchgInst>(
@@ -1751,6 +2117,22 @@ Context::createGetElementPtrInst(llvm::GetElementPtrInst *I) {
       std::unique_ptr<GetElementPtrInst>(new GetElementPtrInst(I, *this));
   return cast<GetElementPtrInst>(registerValue(std::move(NewPtr)));
 }
+SwitchInst *Context::createSwitchInst(llvm::SwitchInst *I) {
+  auto NewPtr = std::unique_ptr<SwitchInst>(new SwitchInst(I, *this));
+  return cast<SwitchInst>(registerValue(std::move(NewPtr)));
+}
+UnaryOperator *Context::createUnaryOperator(llvm::UnaryOperator *I) {
+  auto NewPtr = std::unique_ptr<UnaryOperator>(new UnaryOperator(I, *this));
+  return cast<UnaryOperator>(registerValue(std::move(NewPtr)));
+}
+BinaryOperator *Context::createBinaryOperator(llvm::BinaryOperator *I) {
+  auto NewPtr = std::unique_ptr<BinaryOperator>(new BinaryOperator(I, *this));
+  return cast<BinaryOperator>(registerValue(std::move(NewPtr)));
+}
+AtomicRMWInst *Context::createAtomicRMWInst(llvm::AtomicRMWInst *I) {
+  auto NewPtr = std::unique_ptr<AtomicRMWInst>(new AtomicRMWInst(I, *this));
+  return cast<AtomicRMWInst>(registerValue(std::move(NewPtr)));
+}
 AtomicCmpXchgInst *
 Context::createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I) {
   auto NewPtr =
diff --git a/llvm/lib/SandboxIR/Tracker.cpp b/llvm/lib/SandboxIR/Tracker.cpp
index 23f4125dba026ba..8faff72bc3e2fba 100644
--- a/llvm/lib/SandboxIR/Tracker.cpp
+++ b/llvm/lib/SandboxIR/Tracker.cpp
@@ -27,32 +27,6 @@ void UseSwap::dump() const {
 }
 #endif // NDEBUG
 
-PHISetIncoming::PHISetIncoming(PHINode *PHI, unsigned Idx, What What)
-    : PHI(PHI), Idx(Idx) {
-  switch (What) {
-  case What::Value:
-    OrigValueOrBB = PHI->getIncomingValue(Idx);
-    break;
-  case What::Block:
-    OrigValueOrBB = PHI->getIncomingBlock(Idx);
-    break;
-  }
-}
-
-void PHISetIncoming::revert(Tracker &Tracker) {
-  if (auto *V = OrigValueOrBB.dyn_cast<Value *>())
-    PHI->setIncomingValue(Idx, V);
-  else
-    PHI->setIncomingBlock(Idx, OrigValueOrBB.get<BasicBlock *>());
-}
-
-#ifndef NDEBUG
-void PHISetIncoming::dump() const {
-  dump(dbgs());
-  dbgs() << "\n";
-}
-#endif // NDEBUG
-
 PHIRemoveIncoming::PHIRemoveIncoming(PHINode *PHI, unsigned RemovedIdx)
     : PHI(PHI), RemovedIdx(RemovedIdx) {
   RemovedV = PHI->getIncomingValue(RemovedIdx);
@@ -186,20 +160,26 @@ void RemoveFromParent::dump() const {
 }
 #endif
 
-CallBrInstSetIndirectDest::CallBrInstSetIndirectDest(CallBrInst *CallBr,
-                                                     unsigned Idx)
-    : CallBr(CallBr), Idx(Idx) {
-  OrigIndirectDest = CallBr->getIndirectDest(Idx);
+void SwitchRemoveCase::revert(Tracker &Tracker) { Switch->addCase(Val, Dest); }
+
+#ifndef NDEBUG
+void SwitchRemoveCase::dump() const {
+  dump(dbgs());
+  dbgs() << "\n";
 }
-void CallBrInstSetIndirectDest::revert(Tracker &Tracker) {
-  CallBr->setIndirectDest(Idx, OrigIndirectDest);
+#endif // NDEBUG
+
+void SwitchAddCase::revert(Tracker &Tracker) {
+  auto It = Switch->findCaseValue(Val);
+  Switch->removeCase(It);
 }
+
 #ifndef NDEBUG
-void CallBrInstSetIndirectDest::dump() const {
+void SwitchAddCase::dump() const {
   dump(dbgs());
   dbgs() << "\n";
 }
-#endif
+#endif // NDEBUG
 
 MoveInstr::MoveInstr(Instruction *MovedI) : MovedI(MovedI) {
   if (auto *NextI = MovedI->getNextNode())
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index f653379e3033494..a73ac54a01c5a51 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -168,6 +168,7 @@ add_llvm_component_library(LLVMSupport
   Debug.cpp
   DebugCounter.cpp
   DeltaAlgorithm.cpp
+  DeltaTree.cpp
   DivisionByConstantInfo.cpp
   DAGDeltaAlgorithm.cpp
   DJB.cpp
@@ -216,6 +217,8 @@ add_llvm_component_library(LLVMSupport
   PrettyStackTrace.cpp
   RandomNumberGenerator.cpp
   Regex.cpp
+  RewriteBuffer.cpp
+  RewriteRope.cpp
   RISCVAttributes.cpp
   RISCVAttributeParser.cpp
   RISCVISAUtils.cpp
diff --git a/clang/lib/Rewrite/DeltaTree.cpp b/llvm/lib/Support/DeltaTree.cpp
similarity index 68%
rename from clang/lib/Rewrite/DeltaTree.cpp
rename to llvm/lib/Support/DeltaTree.cpp
index 61467f84928f2aa..2eb642ffd1df6c9 100644
--- a/clang/lib/Rewrite/DeltaTree.cpp
+++ b/llvm/lib/Support/DeltaTree.cpp
@@ -10,13 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Core/DeltaTree.h"
-#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DeltaTree.h"
 #include "llvm/Support/Casting.h"
 #include <cassert>
 #include <cstring>
 
-using namespace clang;
+using namespace llvm;
 
 /// The DeltaTree class is a multiway search tree (BTree) structure with some
 /// fancy features.  B-Trees are generally more memory and cache efficient
@@ -35,125 +34,125 @@ using namespace clang;
 
 namespace {
 
-  /// SourceDelta - As code in the original input buffer is added and deleted,
-  /// SourceDelta records are used to keep track of how the input SourceLocation
-  /// object is mapped into the output buffer.
-  struct SourceDelta {
-    unsigned FileLoc;
-    int Delta;
-
-    static SourceDelta get(unsigned Loc, int D) {
-      SourceDelta Delta;
-      Delta.FileLoc = Loc;
-      Delta.Delta = D;
-      return Delta;
-    }
+/// SourceDelta - As code in the original input buffer is added and deleted,
+/// SourceDelta records are used to keep track of how the input SourceLocation
+/// object is mapped into the output buffer.
+struct SourceDelta {
+  unsigned FileLoc;
+  int Delta;
+
+  static SourceDelta get(unsigned Loc, int D) {
+    SourceDelta Delta;
+    Delta.FileLoc = Loc;
+    Delta.Delta = D;
+    return Delta;
+  }
+};
+
+/// DeltaTreeNode - The common part of all nodes.
+///
+class DeltaTreeNode {
+public:
+  struct InsertResult {
+    DeltaTreeNode *LHS, *RHS;
+    SourceDelta Split;
   };
 
-  /// DeltaTreeNode - The common part of all nodes.
-  ///
-  class DeltaTreeNode {
-  public:
-    struct InsertResult {
-      DeltaTreeNode *LHS, *RHS;
-      SourceDelta Split;
-    };
-
-  private:
-    friend class DeltaTreeInteriorNode;
-
-    /// WidthFactor - This controls the number of K/V slots held in the BTree:
-    /// how wide it is.  Each level of the BTree is guaranteed to have at least
-    /// WidthFactor-1 K/V pairs (except the root) and may have at most
-    /// 2*WidthFactor-1 K/V pairs.
-    enum { WidthFactor = 8 };
-
-    /// Values - This tracks the SourceDelta's currently in this node.
-    SourceDelta Values[2*WidthFactor-1];
-
-    /// NumValuesUsed - This tracks the number of values this node currently
-    /// holds.
-    unsigned char NumValuesUsed = 0;
-
-    /// IsLeaf - This is true if this is a leaf of the btree.  If false, this is
-    /// an interior node, and is actually an instance of DeltaTreeInteriorNode.
-    bool IsLeaf;
-
-    /// FullDelta - This is the full delta of all the values in this node and
-    /// all children nodes.
-    int FullDelta = 0;
+private:
+  friend class DeltaTreeInteriorNode;
 
-  public:
-    DeltaTreeNode(bool isLeaf = true) : IsLeaf(isLeaf) {}
+  /// WidthFactor - This controls the number of K/V slots held in the BTree:
+  /// how wide it is.  Each level of the BTree is guaranteed to have at least
+  /// WidthFactor-1 K/V pairs (except the root) and may have at most
+  /// 2*WidthFactor-1 K/V pairs.
+  enum { WidthFactor = 8 };
 
-    bool isLeaf() const { return IsLeaf; }
-    int getFullDelta() const { return FullDelta; }
-    bool isFull() const { return NumValuesUsed == 2*WidthFactor-1; }
+  /// Values - This tracks the SourceDelta's currently in this node.
+  SourceDelta Values[2 * WidthFactor - 1];
 
-    unsigned getNumValuesUsed() const { return NumValuesUsed; }
+  /// NumValuesUsed - This tracks the number of values this node currently
+  /// holds.
+  unsigned char NumValuesUsed = 0;
 
-    const SourceDelta &getValue(unsigned i) const {
-      assert(i < NumValuesUsed && "Invalid value #");
-      return Values[i];
-    }
+  /// IsLeaf - This is true if this is a leaf of the btree.  If false, this is
+  /// an interior node, and is actually an instance of DeltaTreeInteriorNode.
+  bool IsLeaf;
 
-    SourceDelta &getValue(unsigned i) {
-      assert(i < NumValuesUsed && "Invalid value #");
-      return Values[i];
-    }
+  /// FullDelta - This is the full delta of all the values in this node and
+  /// all children nodes.
+  int FullDelta = 0;
 
-    /// DoInsertion - Do an insertion of the specified FileIndex/Delta pair into
-    /// this node.  If insertion is easy, do it and return false.  Otherwise,
-    /// split the node, populate InsertRes with info about the split, and return
-    /// true.
-    bool DoInsertion(unsigned FileIndex, int Delta, InsertResult *InsertRes);
+public:
+  DeltaTreeNode(bool isLeaf = true) : IsLeaf(isLeaf) {}
 
-    void DoSplit(InsertResult &InsertRes);
+  bool isLeaf() const { return IsLeaf; }
+  int getFullDelta() const { return FullDelta; }
+  bool isFull() const { return NumValuesUsed == 2 * WidthFactor - 1; }
 
+  unsigned getNumValuesUsed() const { return NumValuesUsed; }
 
-    /// RecomputeFullDeltaLocally - Recompute the FullDelta field by doing a
-    /// local walk over our contained deltas.
-    void RecomputeFullDeltaLocally();
+  const SourceDelta &getValue(unsigned i) const {
+    assert(i < NumValuesUsed && "Invalid value #");
+    return Values[i];
+  }
 
-    void Destroy();
-  };
+  SourceDelta &getValue(unsigned i) {
+    assert(i < NumValuesUsed && "Invalid value #");
+    return Values[i];
+  }
 
-  /// DeltaTreeInteriorNode - When isLeaf = false, a node has child pointers.
-  /// This class tracks them.
-  class DeltaTreeInteriorNode : public DeltaTreeNode {
-    friend class DeltaTreeNode;
+  /// DoInsertion - Do an insertion of the specified FileIndex/Delta pair into
+  /// this node.  If insertion is easy, do it and return false.  Otherwise,
+  /// split the node, populate InsertRes with info about the split, and return
+  /// true.
+  bool DoInsertion(unsigned FileIndex, int Delta, InsertResult *InsertRes);
 
-    DeltaTreeNode *Children[2*WidthFactor];
+  void DoSplit(InsertResult &InsertRes);
 
-    ~DeltaTreeInteriorNode() {
-      for (unsigned i = 0, e = NumValuesUsed+1; i != e; ++i)
-        Children[i]->Destroy();
-    }
+  /// RecomputeFullDeltaLocally - Recompute the FullDelta field by doing a
+  /// local walk over our contained deltas.
+  void RecomputeFullDeltaLocally();
 
-  public:
-    DeltaTreeInteriorNode() : DeltaTreeNode(false /*nonleaf*/) {}
+  void Destroy();
+};
 
-    DeltaTreeInteriorNode(const InsertResult &IR)
-        : DeltaTreeNode(false /*nonleaf*/) {
-      Children[0] = IR.LHS;
-      Children[1] = IR.RHS;
-      Values[0] = IR.Split;
-      FullDelta = IR.LHS->getFullDelta()+IR.RHS->getFullDelta()+IR.Split.Delta;
-      NumValuesUsed = 1;
-    }
+/// DeltaTreeInteriorNode - When isLeaf = false, a node has child pointers.
+/// This class tracks them.
+class DeltaTreeInteriorNode : public DeltaTreeNode {
+  friend class DeltaTreeNode;
 
-    const DeltaTreeNode *getChild(unsigned i) const {
-      assert(i < getNumValuesUsed()+1 && "Invalid child");
-      return Children[i];
-    }
+  DeltaTreeNode *Children[2 * WidthFactor];
 
-    DeltaTreeNode *getChild(unsigned i) {
-      assert(i < getNumValuesUsed()+1 && "Invalid child");
-      return Children[i];
-    }
+  ~DeltaTreeInteriorNode() {
+    for (unsigned i = 0, e = NumValuesUsed + 1; i != e; ++i)
+      Children[i]->Destroy();
+  }
 
-    static bool classof(const DeltaTreeNode *N) { return !N->isLeaf(); }
-  };
+public:
+  DeltaTreeInteriorNode() : DeltaTreeNode(false /*nonleaf*/) {}
+
+  DeltaTreeInteriorNode(const InsertResult &IR)
+      : DeltaTreeNode(false /*nonleaf*/) {
+    Children[0] = IR.LHS;
+    Children[1] = IR.RHS;
+    Values[0] = IR.Split;
+    FullDelta =
+        IR.LHS->getFullDelta() + IR.RHS->getFullDelta() + IR.Split.Delta;
+    NumValuesUsed = 1;
+  }
+
+  const DeltaTreeNode *getChild(unsigned i) const {
+    assert(i < getNumValuesUsed() + 1 && "Invalid child");
+    return Children[i];
+  }
+
+  DeltaTreeNode *getChild(unsigned i) {
+    assert(i < getNumValuesUsed() + 1 && "Invalid child");
+    return Children[i];
+  }
+
+  static bool classof(const DeltaTreeNode *N) { return !N->isLeaf(); }
+};
 
 } // namespace
 
@@ -172,7 +171,7 @@ void DeltaTreeNode::RecomputeFullDeltaLocally() {
   for (unsigned i = 0, e = getNumValuesUsed(); i != e; ++i)
     NewFullDelta += Values[i].Delta;
   if (auto *IN = dyn_cast<DeltaTreeInteriorNode>(this))
-    for (unsigned i = 0, e = getNumValuesUsed()+1; i != e; ++i)
+    for (unsigned i = 0, e = getNumValuesUsed() + 1; i != e; ++i)
       NewFullDelta += IN->getChild(i)->getFullDelta();
   FullDelta = NewFullDelta;
 }
@@ -209,7 +208,7 @@ bool DeltaTreeNode::DoInsertion(unsigned FileIndex, int Delta,
       // For an insertion into a non-full leaf node, just insert the value in
       // its sorted position.  This requires moving later values over.
       if (i != e)
-        memmove(&Values[i+1], &Values[i], sizeof(Values[0])*(e-i));
+        memmove(&Values[i + 1], &Values[i], sizeof(Values[0]) * (e - i));
       Values[i] = SourceDelta::get(FileIndex, Delta);
       ++NumValuesUsed;
       return false;
@@ -239,13 +238,13 @@ bool DeltaTreeNode::DoInsertion(unsigned FileIndex, int Delta,
     // into ourself by moving all the later values/children down, then inserting
     // the new one.
     if (i != e)
-      memmove(&IN->Children[i+2], &IN->Children[i+1],
-              (e-i)*sizeof(IN->Children[0]));
+      memmove(&IN->Children[i + 2], &IN->Children[i + 1],
+              (e - i) * sizeof(IN->Children[0]));
     IN->Children[i] = InsertRes->LHS;
-    IN->Children[i+1] = InsertRes->RHS;
+    IN->Children[i + 1] = InsertRes->RHS;
 
     if (e != i)
-      memmove(&Values[i+1], &Values[i], (e-i)*sizeof(Values[0]));
+      memmove(&Values[i + 1], &Values[i], (e - i) * sizeof(Values[0]));
     Values[i] = InsertRes->Split;
     ++NumValuesUsed;
     return false;
@@ -272,20 +271,21 @@ bool DeltaTreeNode::DoInsertion(unsigned FileIndex, int Delta,
   // SubRHS/SubSplit into.  Find out where to insert SubSplit.
 
   // Find the insertion point, the first delta whose index is >SubSplit.FileLoc.
-  i = 0; e = InsertSide->getNumValuesUsed();
+  i = 0;
+  e = InsertSide->getNumValuesUsed();
   while (i != e && SubSplit.FileLoc > InsertSide->getValue(i).FileLoc)
     ++i;
 
   // Now we know that i is the place to insert the split value into.  Insert it
   // and the child right after it.
   if (i != e)
-    memmove(&InsertSide->Children[i+2], &InsertSide->Children[i+1],
-            (e-i)*sizeof(IN->Children[0]));
-  InsertSide->Children[i+1] = SubRHS;
+    memmove(&InsertSide->Children[i + 2], &InsertSide->Children[i + 1],
+            (e - i) * sizeof(IN->Children[0]));
+  InsertSide->Children[i + 1] = SubRHS;
 
   if (e != i)
-    memmove(&InsertSide->Values[i+1], &InsertSide->Values[i],
-            (e-i)*sizeof(Values[0]));
+    memmove(&InsertSide->Values[i + 1], &InsertSide->Values[i],
+            (e - i) * sizeof(Values[0]));
   InsertSide->Values[i] = SubSplit;
   ++InsertSide->NumValuesUsed;
   InsertSide->FullDelta += SubSplit.Delta + SubRHS->getFullDelta();
@@ -310,7 +310,7 @@ void DeltaTreeNode::DoSplit(InsertResult &InsertRes) {
     // into the new node.
     DeltaTreeInteriorNode *New = new DeltaTreeInteriorNode();
     memcpy(&New->Children[0], &IN->Children[WidthFactor],
-           WidthFactor*sizeof(IN->Children[0]));
+           WidthFactor * sizeof(IN->Children[0]));
     NewNode = New;
   } else {
     // Just create the new leaf node.
@@ -319,10 +319,10 @@ void DeltaTreeNode::DoSplit(InsertResult &InsertRes) {
 
   // Move over the last 'WidthFactor-1' values from here to NewNode.
   memcpy(&NewNode->Values[0], &Values[WidthFactor],
-         (WidthFactor-1)*sizeof(Values[0]));
+         (WidthFactor - 1) * sizeof(Values[0]));
 
   // Decrease the number of values in the two nodes.
-  NewNode->NumValuesUsed = NumValuesUsed = WidthFactor-1;
+  NewNode->NumValuesUsed = NumValuesUsed = WidthFactor - 1;
 
   // Recompute the two nodes' full delta.
   NewNode->RecomputeFullDeltaLocally();
@@ -330,14 +330,14 @@ void DeltaTreeNode::DoSplit(InsertResult &InsertRes) {
 
   InsertRes.LHS = this;
   InsertRes.RHS = NewNode;
-  InsertRes.Split = Values[WidthFactor-1];
+  InsertRes.Split = Values[WidthFactor - 1];
 }
 
 //===----------------------------------------------------------------------===//
 //                        DeltaTree Implementation
 //===----------------------------------------------------------------------===//
 
-//#define VERIFY_TREE
+// #define VERIFY_TREE
 
 #ifdef VERIFY_TREE
 /// VerifyTree - Walk the btree performing assertions on various properties to
@@ -350,7 +350,7 @@ static void VerifyTree(const DeltaTreeNode *N) {
     int FullDelta = 0;
     for (unsigned i = 0, e = N->getNumValuesUsed(); i != e; ++i) {
       if (i)
-        assert(N->getValue(i-1).FileLoc < N->getValue(i).FileLoc);
+        assert(N->getValue(i - 1).FileLoc < N->getValue(i).FileLoc);
       FullDelta += N->getValue(i).Delta;
     }
     assert(FullDelta == N->getFullDelta());
@@ -364,16 +364,16 @@ static void VerifyTree(const DeltaTreeNode *N) {
     const SourceDelta &IVal = N->getValue(i);
     const DeltaTreeNode *IChild = IN->getChild(i);
     if (i)
-      assert(IN->getValue(i-1).FileLoc < IVal.FileLoc);
+      assert(IN->getValue(i - 1).FileLoc < IVal.FileLoc);
     FullDelta += IVal.Delta;
     FullDelta += IChild->getFullDelta();
 
     // The largest value in child #i should be smaller than FileLoc.
-    assert(IChild->getValue(IChild->getNumValuesUsed()-1).FileLoc <
+    assert(IChild->getValue(IChild->getNumValuesUsed() - 1).FileLoc <
            IVal.FileLoc);
 
     // The smallest value in child #i+1 should be larger than FileLoc.
-    assert(IN->getChild(i+1)->getValue(0).FileLoc > IVal.FileLoc);
+    assert(IN->getChild(i + 1)->getValue(0).FileLoc > IVal.FileLoc);
     VerifyTree(IChild);
   }
 
@@ -381,15 +381,11 @@ static void VerifyTree(const DeltaTreeNode *N) {
 
   assert(FullDelta == N->getFullDelta());
 }
-#endif  // VERIFY_TREE
+#endif // VERIFY_TREE
 
-static DeltaTreeNode *getRoot(void *Root) {
-  return (DeltaTreeNode*)Root;
-}
+static DeltaTreeNode *getRoot(void *Root) { return (DeltaTreeNode *)Root; }
 
-DeltaTree::DeltaTree() {
-  Root = new DeltaTreeNode();
-}
+DeltaTree::DeltaTree() { Root = new DeltaTreeNode(); }
 
 DeltaTree::DeltaTree(const DeltaTree &RHS) {
   // Currently we only support copying when the RHS is empty.
@@ -398,9 +394,7 @@ DeltaTree::DeltaTree(const DeltaTree &RHS) {
   Root = new DeltaTreeNode();
 }
 
-DeltaTree::~DeltaTree() {
-  getRoot(Root)->Destroy();
-}
+DeltaTree::~DeltaTree() { getRoot(Root)->Destroy(); }
 
 /// getDeltaAt - Return the accumulated delta at the specified file offset.
 /// This includes all insertions or delections that occurred *before* the
@@ -428,7 +422,8 @@ int DeltaTree::getDeltaAt(unsigned FileIndex) const {
     // If we have an interior node, include information about children and
     // recurse.  Otherwise, if we have a leaf, we're done.
     const auto *IN = dyn_cast<DeltaTreeInteriorNode>(Node);
-    if (!IN) return Result;
+    if (!IN)
+      return Result;
 
     // Include any children to the left of the values we skipped, all of
     // their deltas should be included as well.
@@ -440,7 +435,7 @@ int DeltaTree::getDeltaAt(unsigned FileIndex) const {
     // partial results.
     if (NumValsGreater != Node->getNumValuesUsed() &&
         Node->getValue(NumValsGreater).FileLoc == FileIndex)
-      return Result+IN->getChild(NumValsGreater)->getFullDelta();
+      return Result + IN->getChild(NumValsGreater)->getFullDelta();
 
     // Otherwise, traverse down the tree.  The selected subtree may be
     // partially included in the range.
diff --git a/llvm/lib/Support/FormatVariadic.cpp b/llvm/lib/Support/FormatVariadic.cpp
index 3c07a80a00ae6d7..e25d036cdf1e8c3 100644
--- a/llvm/lib/Support/FormatVariadic.cpp
+++ b/llvm/lib/Support/FormatVariadic.cpp
@@ -107,15 +107,18 @@ formatv_object_base::splitLiteralAndReplacement(StringRef Fmt) {
       StringRef Right = Fmt.drop_front(NumEscapedBraces * 2);
       return std::make_pair(ReplacementItem{Middle}, Right);
     }
-    // An unterminated open brace is undefined.  We treat the rest of the string
-    // as a literal replacement, but we assert to indicate that this is
-    // undefined and that we consider it an error.
+    // An unterminated open brace is undefined. Assert to indicate that this is
+    // undefined and that we consider it an error. When asserts are disabled,
+    // build a replacement item with an error message.
     std::size_t BC = Fmt.find_first_of('}');
     if (BC == StringRef::npos) {
       assert(
           false &&
-          "Unterminated brace sequence.  Escape with {{ for a literal brace.");
-      return std::make_pair(ReplacementItem{Fmt}, StringRef());
+          "Unterminated brace sequence. Escape with {{ for a literal brace.");
+      return std::make_pair(
+          ReplacementItem{"Unterminated brace sequence. Escape with {{ for a "
+                          "literal brace."},
+          StringRef());
     }
 
     // Even if there is a closing brace, if there is another open brace before
diff --git a/llvm/lib/Support/RewriteBuffer.cpp b/llvm/lib/Support/RewriteBuffer.cpp
new file mode 100644
index 000000000000000..cc6ba7e97b1a7fd
--- /dev/null
+++ b/llvm/lib/Support/RewriteBuffer.cpp
@@ -0,0 +1,107 @@
+//===- RewriteBuffer.h - Buffer rewriting interface -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/RewriteBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+raw_ostream &RewriteBuffer::write(raw_ostream &Stream) const {
+  // Walk RewriteRope chunks efficiently using MoveToNextPiece() instead of the
+  // character iterator.
+  for (RopePieceBTreeIterator I = begin(), E = end(); I != E;
+       I.MoveToNextPiece())
+    Stream << I.piece();
+  return Stream;
+}
+
+/// Return true if this character is non-new-line whitespace:
+/// ' ', '\\t', '\\f', '\\v', '\\r'.
+static inline bool isWhitespaceExceptNL(unsigned char c) {
+  return c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r';
+}
+
+void RewriteBuffer::RemoveText(unsigned OrigOffset, unsigned Size,
+                               bool removeLineIfEmpty) {
+  // Nothing to remove, exit early.
+  if (Size == 0)
+    return;
+
+  unsigned RealOffset = getMappedOffset(OrigOffset, true);
+  assert(RealOffset + Size <= Buffer.size() && "Invalid location");
+
+  // Remove the dead characters.
+  Buffer.erase(RealOffset, Size);
+
+  // Add a delta so that future changes are offset correctly.
+  AddReplaceDelta(OrigOffset, -Size);
+
+  if (removeLineIfEmpty) {
+    // Find the line that the remove occurred and if it is completely empty
+    // remove the line as well.
+
+    iterator curLineStart = begin();
+    unsigned curLineStartOffs = 0;
+    iterator posI = begin();
+    for (unsigned i = 0; i != RealOffset; ++i) {
+      if (*posI == '\n') {
+        curLineStart = posI;
+        ++curLineStart;
+        curLineStartOffs = i + 1;
+      }
+      ++posI;
+    }
+
+    unsigned lineSize = 0;
+    posI = curLineStart;
+    while (posI != end() && isWhitespaceExceptNL(*posI)) {
+      ++posI;
+      ++lineSize;
+    }
+    if (posI != end() && *posI == '\n') {
+      Buffer.erase(curLineStartOffs, lineSize + 1 /* + '\n'*/);
+      // FIXME: Here, the offset of the start of the line is supposed to be
+      // expressed in terms of the original input not the "real" rewrite
+      // buffer.  How do we compute that reliably?  It might be tempting to use
+      // curLineStartOffs + OrigOffset - RealOffset, but that assumes the
+      // difference between the original and real offset is the same at the
+      // removed text and at the start of the line, but that's not true if
+      // edits were previously made earlier on the line.  This bug is also
+      // documented by a FIXME on the definition of
+      // clang::Rewriter::RewriteOptions::RemoveLineIfEmpty.  A reproducer for
+      // the implementation below is the test RemoveLineIfEmpty in
+      // clang/unittests/Rewrite/RewriteBufferTest.cpp.
+      AddReplaceDelta(curLineStartOffs, -(lineSize + 1 /* + '\n'*/));
+    }
+  }
+}
+
+void RewriteBuffer::InsertText(unsigned OrigOffset, StringRef Str,
+                               bool InsertAfter) {
+  // Nothing to insert, exit early.
+  if (Str.empty())
+    return;
+
+  unsigned RealOffset = getMappedOffset(OrigOffset, InsertAfter);
+  Buffer.insert(RealOffset, Str.begin(), Str.end());
+
+  // Add a delta so that future changes are offset correctly.
+  AddInsertDelta(OrigOffset, Str.size());
+}
+
+/// ReplaceText - This method replaces a range of characters in the input
+/// buffer with a new string.  This is effectively a combined "remove+insert"
+/// operation.
+void RewriteBuffer::ReplaceText(unsigned OrigOffset, unsigned OrigLength,
+                                StringRef NewStr) {
+  unsigned RealOffset = getMappedOffset(OrigOffset, true);
+  Buffer.erase(RealOffset, OrigLength);
+  Buffer.insert(RealOffset, NewStr.begin(), NewStr.end());
+  if (OrigLength != NewStr.size())
+    AddReplaceDelta(OrigOffset, NewStr.size() - OrigLength);
+}
diff --git a/clang/lib/Rewrite/RewriteRope.cpp b/llvm/lib/Support/RewriteRope.cpp
similarity index 63%
rename from clang/lib/Rewrite/RewriteRope.cpp
rename to llvm/lib/Support/RewriteRope.cpp
index 980d0f01e277c4d..7440aa6ab501e78 100644
--- a/clang/lib/Rewrite/RewriteRope.cpp
+++ b/llvm/lib/Support/RewriteRope.cpp
@@ -10,14 +10,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Core/RewriteRope.h"
-#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/RewriteRope.h"
 #include "llvm/Support/Casting.h"
 #include <algorithm>
 #include <cassert>
 #include <cstring>
 
-using namespace clang;
+using namespace llvm;
 
 /// RewriteRope is a "strong" string class, designed to make insertions and
 /// deletions in the middle of the string nearly constant time (really, they are
@@ -68,162 +67,160 @@ namespace {
 // RopePieceBTreeNode Class
 //===----------------------------------------------------------------------===//
 
-  /// RopePieceBTreeNode - Common base class of RopePieceBTreeLeaf and
-  /// RopePieceBTreeInterior.  This provides some 'virtual' dispatching methods
-  /// and a flag that determines which subclass the instance is.  Also
-  /// important, this node knows the full extend of the node, including any
-  /// children that it has.  This allows efficient skipping over entire subtrees
-  /// when looking for an offset in the BTree.
-  class RopePieceBTreeNode {
-  protected:
-    /// WidthFactor - This controls the number of K/V slots held in the BTree:
-    /// how wide it is.  Each level of the BTree is guaranteed to have at least
-    /// 'WidthFactor' elements in it (either ropepieces or children), (except
-    /// the root, which may have less) and may have at most 2*WidthFactor
-    /// elements.
-    enum { WidthFactor = 8 };
-
-    /// Size - This is the number of bytes of file this node (including any
-    /// potential children) covers.
-    unsigned Size = 0;
-
-    /// IsLeaf - True if this is an instance of RopePieceBTreeLeaf, false if it
-    /// is an instance of RopePieceBTreeInterior.
-    bool IsLeaf;
-
-    RopePieceBTreeNode(bool isLeaf) : IsLeaf(isLeaf) {}
-    ~RopePieceBTreeNode() = default;
-
-  public:
-    bool isLeaf() const { return IsLeaf; }
-    unsigned size() const { return Size; }
-
-    void Destroy();
-
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-  };
+/// RopePieceBTreeNode - Common base class of RopePieceBTreeLeaf and
+/// RopePieceBTreeInterior.  This provides some 'virtual' dispatching methods
+/// and a flag that determines which subclass the instance is.  Also
+/// important, this node knows the full extend of the node, including any
+/// children that it has.  This allows efficient skipping over entire subtrees
+/// when looking for an offset in the BTree.
+class RopePieceBTreeNode {
+protected:
+  /// WidthFactor - This controls the number of K/V slots held in the BTree:
+  /// how wide it is.  Each level of the BTree is guaranteed to have at least
+  /// 'WidthFactor' elements in it (either ropepieces or children), (except
+  /// the root, which may have less) and may have at most 2*WidthFactor
+  /// elements.
+  enum { WidthFactor = 8 };
+
+  /// Size - This is the number of bytes of file this node (including any
+  /// potential children) covers.
+  unsigned Size = 0;
+
+  /// IsLeaf - True if this is an instance of RopePieceBTreeLeaf, false if it
+  /// is an instance of RopePieceBTreeInterior.
+  bool IsLeaf;
+
+  RopePieceBTreeNode(bool isLeaf) : IsLeaf(isLeaf) {}
+  ~RopePieceBTreeNode() = default;
+
+public:
+  bool isLeaf() const { return IsLeaf; }
+  unsigned size() const { return Size; }
+
+  void Destroy();
+
+  /// split - Split the range containing the specified offset so that we are
+  /// guaranteed that there is a place to do an insertion at the specified
+  /// offset.  The offset is relative, so "0" is the start of the node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *split(unsigned Offset);
+
+  /// insert - Insert the specified ropepiece into this tree node at the
+  /// specified offset.  The offset is relative, so "0" is the start of the
+  /// node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+  /// erase - Remove NumBytes from this node at the specified offset.  We are
+  /// guaranteed that there is a split at Offset.
+  void erase(unsigned Offset, unsigned NumBytes);
+};
 
 //===----------------------------------------------------------------------===//
 // RopePieceBTreeLeaf Class
 //===----------------------------------------------------------------------===//
 
-  /// RopePieceBTreeLeaf - Directly manages up to '2*WidthFactor' RopePiece
-  /// nodes.  This directly represents a chunk of the string with those
-  /// RopePieces concatenated.  Since this is a B+Tree, all values (in this case
-  /// instances of RopePiece) are stored in leaves like this.  To make iteration
-  /// over the leaves efficient, they maintain a singly linked list through the
-  /// NextLeaf field.  This allows the B+Tree forward iterator to be constant
-  /// time for all increments.
-  class RopePieceBTreeLeaf : public RopePieceBTreeNode {
-    /// NumPieces - This holds the number of rope pieces currently active in the
-    /// Pieces array.
-    unsigned char NumPieces = 0;
-
-    /// Pieces - This tracks the file chunks currently in this leaf.
-    RopePiece Pieces[2*WidthFactor];
-
-    /// NextLeaf - This is a pointer to the next leaf in the tree, allowing
-    /// efficient in-order forward iteration of the tree without traversal.
-    RopePieceBTreeLeaf **PrevLeaf = nullptr;
-    RopePieceBTreeLeaf *NextLeaf = nullptr;
-
-  public:
-    RopePieceBTreeLeaf() : RopePieceBTreeNode(true) {}
-
-    ~RopePieceBTreeLeaf() {
-      if (PrevLeaf || NextLeaf)
-        removeFromLeafInOrder();
-      clear();
-    }
+/// RopePieceBTreeLeaf - Directly manages up to '2*WidthFactor' RopePiece
+/// nodes.  This directly represents a chunk of the string with those
+/// RopePieces concatenated.  Since this is a B+Tree, all values (in this case
+/// instances of RopePiece) are stored in leaves like this.  To make iteration
+/// over the leaves efficient, they maintain a singly linked list through the
+/// NextLeaf field.  This allows the B+Tree forward iterator to be constant
+/// time for all increments.
+class RopePieceBTreeLeaf : public RopePieceBTreeNode {
+  /// NumPieces - This holds the number of rope pieces currently active in the
+  /// Pieces array.
+  unsigned char NumPieces = 0;
+
+  /// Pieces - This tracks the file chunks currently in this leaf.
+  RopePiece Pieces[2 * WidthFactor];
+
+  /// NextLeaf - This is a pointer to the next leaf in the tree, allowing
+  /// efficient in-order forward iteration of the tree without traversal.
+  RopePieceBTreeLeaf **PrevLeaf = nullptr;
+  RopePieceBTreeLeaf *NextLeaf = nullptr;
+
+public:
+  RopePieceBTreeLeaf() : RopePieceBTreeNode(true) {}
+
+  ~RopePieceBTreeLeaf() {
+    if (PrevLeaf || NextLeaf)
+      removeFromLeafInOrder();
+    clear();
+  }
 
-    bool isFull() const { return NumPieces == 2*WidthFactor; }
+  bool isFull() const { return NumPieces == 2 * WidthFactor; }
 
-    /// clear - Remove all rope pieces from this leaf.
-    void clear() {
-      while (NumPieces)
-        Pieces[--NumPieces] = RopePiece();
-      Size = 0;
-    }
+  /// clear - Remove all rope pieces from this leaf.
+  void clear() {
+    while (NumPieces)
+      Pieces[--NumPieces] = RopePiece();
+    Size = 0;
+  }
 
-    unsigned getNumPieces() const { return NumPieces; }
+  unsigned getNumPieces() const { return NumPieces; }
 
-    const RopePiece &getPiece(unsigned i) const {
-      assert(i < getNumPieces() && "Invalid piece ID");
-      return Pieces[i];
-    }
+  const RopePiece &getPiece(unsigned i) const {
+    assert(i < getNumPieces() && "Invalid piece ID");
+    return Pieces[i];
+  }
 
-    const RopePieceBTreeLeaf *getNextLeafInOrder() const { return NextLeaf; }
+  const RopePieceBTreeLeaf *getNextLeafInOrder() const { return NextLeaf; }
 
-    void insertAfterLeafInOrder(RopePieceBTreeLeaf *Node) {
-      assert(!PrevLeaf && !NextLeaf && "Already in ordering");
+  void insertAfterLeafInOrder(RopePieceBTreeLeaf *Node) {
+    assert(!PrevLeaf && !NextLeaf && "Already in ordering");
 
-      NextLeaf = Node->NextLeaf;
-      if (NextLeaf)
-        NextLeaf->PrevLeaf = &NextLeaf;
-      PrevLeaf = &Node->NextLeaf;
-      Node->NextLeaf = this;
-    }
+    NextLeaf = Node->NextLeaf;
+    if (NextLeaf)
+      NextLeaf->PrevLeaf = &NextLeaf;
+    PrevLeaf = &Node->NextLeaf;
+    Node->NextLeaf = this;
+  }
 
-    void removeFromLeafInOrder() {
-      if (PrevLeaf) {
-        *PrevLeaf = NextLeaf;
-        if (NextLeaf)
-          NextLeaf->PrevLeaf = PrevLeaf;
-      } else if (NextLeaf) {
-        NextLeaf->PrevLeaf = nullptr;
-      }
+  void removeFromLeafInOrder() {
+    if (PrevLeaf) {
+      *PrevLeaf = NextLeaf;
+      if (NextLeaf)
+        NextLeaf->PrevLeaf = PrevLeaf;
+    } else if (NextLeaf) {
+      NextLeaf->PrevLeaf = nullptr;
     }
+  }
 
-    /// FullRecomputeSizeLocally - This method recomputes the 'Size' field by
-    /// summing the size of all RopePieces.
-    void FullRecomputeSizeLocally() {
-      Size = 0;
-      for (unsigned i = 0, e = getNumPieces(); i != e; ++i)
-        Size += getPiece(i).size();
-    }
+  /// FullRecomputeSizeLocally - This method recomputes the 'Size' field by
+  /// summing the size of all RopePieces.
+  void FullRecomputeSizeLocally() {
+    Size = 0;
+    for (unsigned i = 0, e = getNumPieces(); i != e; ++i)
+      Size += getPiece(i).size();
+  }
 
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-
-    static bool classof(const RopePieceBTreeNode *N) {
-      return N->isLeaf();
-    }
-  };
+  /// split - Split the range containing the specified offset so that we are
+  /// guaranteed that there is a place to do an insertion at the specified
+  /// offset.  The offset is relative, so "0" is the start of the node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *split(unsigned Offset);
+
+  /// insert - Insert the specified ropepiece into this tree node at the
+  /// specified offset.  The offset is relative, so "0" is the start of the
+  /// node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+  /// erase - Remove NumBytes from this node at the specified offset.  We are
+  /// guaranteed that there is a split at Offset.
+  void erase(unsigned Offset, unsigned NumBytes);
+
+  static bool classof(const RopePieceBTreeNode *N) { return N->isLeaf(); }
+};
 
 } // namespace
 
@@ -244,7 +241,7 @@ RopePieceBTreeNode *RopePieceBTreeLeaf::split(unsigned Offset) {
   // Find the piece that this offset lands in.
   unsigned PieceOffs = 0;
   unsigned i = 0;
-  while (Offset >= PieceOffs+Pieces[i].size()) {
+  while (Offset >= PieceOffs + Pieces[i].size()) {
     PieceOffs += Pieces[i].size();
     ++i;
   }
@@ -256,13 +253,13 @@ RopePieceBTreeNode *RopePieceBTreeLeaf::split(unsigned Offset) {
 
   // Otherwise, we need to split piece 'i' at Offset-PieceOffs.  Convert Offset
   // to being Piece relative.
-  unsigned IntraPieceOffset = Offset-PieceOffs;
+  unsigned IntraPieceOffset = Offset - PieceOffs;
 
   // We do this by shrinking the RopePiece and then doing an insert of the tail.
-  RopePiece Tail(Pieces[i].StrData, Pieces[i].StartOffs+IntraPieceOffset,
+  RopePiece Tail(Pieces[i].StrData, Pieces[i].StartOffs + IntraPieceOffset,
                  Pieces[i].EndOffs);
   Size -= Pieces[i].size();
-  Pieces[i].EndOffs = Pieces[i].StartOffs+IntraPieceOffset;
+  Pieces[i].EndOffs = Pieces[i].StartOffs + IntraPieceOffset;
   Size += Pieces[i].size();
 
   return insert(Offset, Tail);
@@ -293,7 +290,7 @@ RopePieceBTreeNode *RopePieceBTreeLeaf::insert(unsigned Offset,
     // For an insertion into a non-full leaf node, just insert the value in
     // its sorted position.  This requires moving later values over.
     for (; i != e; --e)
-      Pieces[e] = Pieces[e-1];
+      Pieces[e] = Pieces[e - 1];
     Pieces[i] = R;
     ++NumPieces;
     Size += R.size();
@@ -309,10 +306,10 @@ RopePieceBTreeNode *RopePieceBTreeLeaf::insert(unsigned Offset,
   RopePieceBTreeLeaf *NewNode = new RopePieceBTreeLeaf();
 
   // Move over the last 'WidthFactor' values from here to NewNode.
-  std::copy(&Pieces[WidthFactor], &Pieces[2*WidthFactor],
+  std::copy(&Pieces[WidthFactor], &Pieces[2 * WidthFactor],
             &NewNode->Pieces[0]);
   // Replace old pieces with null RopePieces to drop refcounts.
-  std::fill(&Pieces[WidthFactor], &Pieces[2*WidthFactor], RopePiece());
+  std::fill(&Pieces[WidthFactor], &Pieces[2 * WidthFactor], RopePiece());
 
   // Decrease the number of values in the two nodes.
   NewNode->NumPieces = NumPieces = WidthFactor;
@@ -347,33 +344,34 @@ void RopePieceBTreeLeaf::erase(unsigned Offset, unsigned NumBytes) {
 
   // Figure out how many pieces completely cover 'NumBytes'.  We want to remove
   // all of them.
-  for (; Offset+NumBytes > PieceOffs+getPiece(i).size(); ++i)
+  for (; Offset + NumBytes > PieceOffs + getPiece(i).size(); ++i)
     PieceOffs += getPiece(i).size();
 
   // If we exactly include the last one, include it in the region to delete.
-  if (Offset+NumBytes == PieceOffs+getPiece(i).size()) {
+  if (Offset + NumBytes == PieceOffs + getPiece(i).size()) {
     PieceOffs += getPiece(i).size();
     ++i;
   }
 
   // If we completely cover some RopePieces, erase them now.
   if (i != StartPiece) {
-    unsigned NumDeleted = i-StartPiece;
+    unsigned NumDeleted = i - StartPiece;
     for (; i != getNumPieces(); ++i)
-      Pieces[i-NumDeleted] = Pieces[i];
+      Pieces[i - NumDeleted] = Pieces[i];
 
     // Drop references to dead rope pieces.
-    std::fill(&Pieces[getNumPieces()-NumDeleted], &Pieces[getNumPieces()],
+    std::fill(&Pieces[getNumPieces() - NumDeleted], &Pieces[getNumPieces()],
               RopePiece());
     NumPieces -= NumDeleted;
 
-    unsigned CoverBytes = PieceOffs-Offset;
+    unsigned CoverBytes = PieceOffs - Offset;
     NumBytes -= CoverBytes;
     Size -= CoverBytes;
   }
 
   // If we completely removed some stuff, we could be done.
-  if (NumBytes == 0) return;
+  if (NumBytes == 0)
+    return;
 
   // Okay, now might be erasing part of some Piece.  If this is the case, then
   // move the start point of the piece.
@@ -390,81 +388,79 @@ void RopePieceBTreeLeaf::erase(unsigned Offset, unsigned NumBytes) {
 
 namespace {
 
-  /// RopePieceBTreeInterior - This represents an interior node in the B+Tree,
-  /// which holds up to 2*WidthFactor pointers to child nodes.
-  class RopePieceBTreeInterior : public RopePieceBTreeNode {
-    /// NumChildren - This holds the number of children currently active in the
-    /// Children array.
-    unsigned char NumChildren = 0;
+/// RopePieceBTreeInterior - This represents an interior node in the B+Tree,
+/// which holds up to 2*WidthFactor pointers to child nodes.
+class RopePieceBTreeInterior : public RopePieceBTreeNode {
+  /// NumChildren - This holds the number of children currently active in the
+  /// Children array.
+  unsigned char NumChildren = 0;
 
-    RopePieceBTreeNode *Children[2*WidthFactor];
+  RopePieceBTreeNode *Children[2 * WidthFactor];
 
-  public:
-    RopePieceBTreeInterior() : RopePieceBTreeNode(false) {}
+public:
+  RopePieceBTreeInterior() : RopePieceBTreeNode(false) {}
 
-    RopePieceBTreeInterior(RopePieceBTreeNode *LHS, RopePieceBTreeNode *RHS)
-        : RopePieceBTreeNode(false) {
-      Children[0] = LHS;
-      Children[1] = RHS;
-      NumChildren = 2;
-      Size = LHS->size() + RHS->size();
-    }
+  RopePieceBTreeInterior(RopePieceBTreeNode *LHS, RopePieceBTreeNode *RHS)
+      : RopePieceBTreeNode(false) {
+    Children[0] = LHS;
+    Children[1] = RHS;
+    NumChildren = 2;
+    Size = LHS->size() + RHS->size();
+  }
 
-    ~RopePieceBTreeInterior() {
-      for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
-        Children[i]->Destroy();
-    }
+  ~RopePieceBTreeInterior() {
+    for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
+      Children[i]->Destroy();
+  }
 
-    bool isFull() const { return NumChildren == 2*WidthFactor; }
+  bool isFull() const { return NumChildren == 2 * WidthFactor; }
 
-    unsigned getNumChildren() const { return NumChildren; }
+  unsigned getNumChildren() const { return NumChildren; }
 
-    const RopePieceBTreeNode *getChild(unsigned i) const {
-      assert(i < NumChildren && "invalid child #");
-      return Children[i];
-    }
+  const RopePieceBTreeNode *getChild(unsigned i) const {
+    assert(i < NumChildren && "invalid child #");
+    return Children[i];
+  }
 
-    RopePieceBTreeNode *getChild(unsigned i) {
-      assert(i < NumChildren && "invalid child #");
-      return Children[i];
-    }
+  RopePieceBTreeNode *getChild(unsigned i) {
+    assert(i < NumChildren && "invalid child #");
+    return Children[i];
+  }
 
-    /// FullRecomputeSizeLocally - Recompute the Size field of this node by
-    /// summing up the sizes of the child nodes.
-    void FullRecomputeSizeLocally() {
-      Size = 0;
-      for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
-        Size += getChild(i)->size();
-    }
+  /// FullRecomputeSizeLocally - Recompute the Size field of this node by
+  /// summing up the sizes of the child nodes.
+  void FullRecomputeSizeLocally() {
+    Size = 0;
+    for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
+      Size += getChild(i)->size();
+  }
 
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-    /// HandleChildPiece - A child propagated an insertion result up to us.
-    /// Insert the new child, and/or propagate the result further up the tree.
-    RopePieceBTreeNode *HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS);
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-
-    static bool classof(const RopePieceBTreeNode *N) {
-      return !N->isLeaf();
-    }
-  };
+  /// split - Split the range containing the specified offset so that we are
+  /// guaranteed that there is a place to do an insertion at the specified
+  /// offset.  The offset is relative, so "0" is the start of the node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *split(unsigned Offset);
+
+  /// insert - Insert the specified ropepiece into this tree node at the
+  /// specified offset.  The offset is relative, so "0" is the start of the
+  /// node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+  /// HandleChildPiece - A child propagated an insertion result up to us.
+  /// Insert the new child, and/or propagate the result further up the tree.
+  RopePieceBTreeNode *HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS);
+
+  /// erase - Remove NumBytes from this node at the specified offset.  We are
+  /// guaranteed that there is a split at Offset.
+  void erase(unsigned Offset, unsigned NumBytes);
+
+  static bool classof(const RopePieceBTreeNode *N) { return !N->isLeaf(); }
+};
 
 } // namespace
 
@@ -481,7 +477,7 @@ RopePieceBTreeNode *RopePieceBTreeInterior::split(unsigned Offset) {
 
   unsigned ChildOffset = 0;
   unsigned i = 0;
-  for (; Offset >= ChildOffset+getChild(i)->size(); ++i)
+  for (; Offset >= ChildOffset + getChild(i)->size(); ++i)
     ChildOffset += getChild(i)->size();
 
   // If already split there, we're done.
@@ -489,7 +485,7 @@ RopePieceBTreeNode *RopePieceBTreeInterior::split(unsigned Offset) {
     return nullptr;
 
   // Otherwise, recursively split the child.
-  if (RopePieceBTreeNode *RHS = getChild(i)->split(Offset-ChildOffset))
+  if (RopePieceBTreeNode *RHS = getChild(i)->split(Offset - ChildOffset))
     return HandleChildPiece(i, RHS);
   return nullptr; // Done!
 }
@@ -509,17 +505,17 @@ RopePieceBTreeNode *RopePieceBTreeInterior::insert(unsigned Offset,
   unsigned ChildOffs = 0;
   if (Offset == size()) {
     // Fastpath for a common case.  Insert at end of last child.
-    i = e-1;
-    ChildOffs = size()-getChild(i)->size();
+    i = e - 1;
+    ChildOffs = size() - getChild(i)->size();
   } else {
-    for (; Offset > ChildOffs+getChild(i)->size(); ++i)
+    for (; Offset > ChildOffs + getChild(i)->size(); ++i)
       ChildOffs += getChild(i)->size();
   }
 
   Size += R.size();
 
   // Insert at the end of this child.
-  if (RopePieceBTreeNode *RHS = getChild(i)->insert(Offset-ChildOffs, R))
+  if (RopePieceBTreeNode *RHS = getChild(i)->insert(Offset - ChildOffs, R))
     return HandleChildPiece(i, RHS);
 
   return nullptr;
@@ -534,9 +530,9 @@ RopePieceBTreeInterior::HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS) {
   if (!isFull()) {
     // Insert RHS after child 'i'.
     if (i + 1 != getNumChildren())
-      memmove(&Children[i+2], &Children[i+1],
-              (getNumChildren()-i-1)*sizeof(Children[0]));
-    Children[i+1] = RHS;
+      memmove(&Children[i + 2], &Children[i + 1],
+              (getNumChildren() - i - 1) * sizeof(Children[0]));
+    Children[i + 1] = RHS;
     ++NumChildren;
     return nullptr;
   }
@@ -549,7 +545,7 @@ RopePieceBTreeInterior::HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS) {
 
   // Move over the last 'WidthFactor' values from here to NewNode.
   memcpy(&NewNode->Children[0], &Children[WidthFactor],
-         WidthFactor*sizeof(Children[0]));
+         WidthFactor * sizeof(Children[0]));
 
   // Decrease the number of values in the two nodes.
   NewNode->NumChildren = NumChildren = WidthFactor;
@@ -559,7 +555,7 @@ RopePieceBTreeInterior::HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS) {
   if (i < WidthFactor)
     this->HandleChildPiece(i, RHS);
   else
-    NewNode->HandleChildPiece(i-WidthFactor, RHS);
+    NewNode->HandleChildPiece(i - WidthFactor, RHS);
 
   // Recompute the two nodes' size.
   NewNode->FullRecomputeSizeLocally();
@@ -585,7 +581,7 @@ void RopePieceBTreeInterior::erase(unsigned Offset, unsigned NumBytes) {
 
     // If we are deleting something contained entirely in the child, pass on the
     // request.
-    if (Offset+NumBytes < CurChild->size()) {
+    if (Offset + NumBytes < CurChild->size()) {
       CurChild->erase(Offset, NumBytes);
       return;
     }
@@ -593,7 +589,7 @@ void RopePieceBTreeInterior::erase(unsigned Offset, unsigned NumBytes) {
     // If this deletion request starts somewhere in the middle of the child, it
     // must be deleting to the end of the child.
     if (Offset) {
-      unsigned BytesFromChild = CurChild->size()-Offset;
+      unsigned BytesFromChild = CurChild->size() - Offset;
       CurChild->erase(Offset, BytesFromChild);
       NumBytes -= BytesFromChild;
       // Start at the beginning of the next child.
@@ -608,8 +604,8 @@ void RopePieceBTreeInterior::erase(unsigned Offset, unsigned NumBytes) {
     CurChild->Destroy();
     --NumChildren;
     if (i != getNumChildren())
-      memmove(&Children[i], &Children[i+1],
-              (getNumChildren()-i)*sizeof(Children[0]));
+      memmove(&Children[i], &Children[i + 1],
+              (getNumChildren() - i) * sizeof(Children[0]));
   }
 }
 
@@ -654,7 +650,7 @@ RopePieceBTreeNode *RopePieceBTreeNode::insert(unsigned Offset,
 /// erase - Remove NumBytes from this node at the specified offset.  We are
 /// guaranteed that there is a split at Offset.
 void RopePieceBTreeNode::erase(unsigned Offset, unsigned NumBytes) {
-  assert(Offset+NumBytes <= size() && "Invalid offset to erase!");
+  assert(Offset + NumBytes <= size() && "Invalid offset to erase!");
   if (auto *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
     return Leaf->erase(Offset, NumBytes);
   return cast<RopePieceBTreeInterior>(this)->erase(Offset, NumBytes);
@@ -665,7 +661,7 @@ void RopePieceBTreeNode::erase(unsigned Offset, unsigned NumBytes) {
 //===----------------------------------------------------------------------===//
 
 static const RopePieceBTreeLeaf *getCN(const void *P) {
-  return static_cast<const RopePieceBTreeLeaf*>(P);
+  return static_cast<const RopePieceBTreeLeaf *>(P);
 }
 
 // begin iterator.
@@ -686,13 +682,14 @@ RopePieceBTreeIterator::RopePieceBTreeIterator(const void *n) {
 
   if (CurNode)
     CurPiece = &getCN(CurNode)->getPiece(0);
-  else  // Empty tree, this is an end() iterator.
+  else // Empty tree, this is an end() iterator.
     CurPiece = nullptr;
   CurChar = 0;
 }
 
 void RopePieceBTreeIterator::MoveToNextPiece() {
-  if (CurPiece != &getCN(CurNode)->getPiece(getCN(CurNode)->getNumPieces()-1)) {
+  if (CurPiece !=
+      &getCN(CurNode)->getPiece(getCN(CurNode)->getNumPieces() - 1)) {
     CurChar = 0;
     ++CurPiece;
     return;
@@ -715,25 +712,19 @@ void RopePieceBTreeIterator::MoveToNextPiece() {
 //===----------------------------------------------------------------------===//
 
 static RopePieceBTreeNode *getRoot(void *P) {
-  return static_cast<RopePieceBTreeNode*>(P);
+  return static_cast<RopePieceBTreeNode *>(P);
 }
 
-RopePieceBTree::RopePieceBTree() {
-  Root = new RopePieceBTreeLeaf();
-}
+RopePieceBTree::RopePieceBTree() { Root = new RopePieceBTreeLeaf(); }
 
 RopePieceBTree::RopePieceBTree(const RopePieceBTree &RHS) {
   assert(RHS.empty() && "Can't copy non-empty tree yet");
   Root = new RopePieceBTreeLeaf();
 }
 
-RopePieceBTree::~RopePieceBTree() {
-  getRoot(Root)->Destroy();
-}
+RopePieceBTree::~RopePieceBTree() { getRoot(Root)->Destroy(); }
 
-unsigned RopePieceBTree::size() const {
-  return getRoot(Root)->size();
-}
+unsigned RopePieceBTree::size() const { return getRoot(Root)->size(); }
 
 void RopePieceBTree::clear() {
   if (auto *Leaf = dyn_cast<RopePieceBTreeLeaf>(getRoot(Root)))
@@ -772,24 +763,24 @@ void RopePieceBTree::erase(unsigned Offset, unsigned NumBytes) {
 /// the AllocBuffer object to aggregate requests for small strings into one
 /// allocation instead of doing tons of tiny allocations.
 RopePiece RewriteRope::MakeRopeString(const char *Start, const char *End) {
-  unsigned Len = End-Start;
+  unsigned Len = End - Start;
   assert(Len && "Zero length RopePiece is invalid!");
 
   // If we have space for this string in the current alloc buffer, use it.
-  if (AllocOffs+Len <= AllocChunkSize) {
-    memcpy(AllocBuffer->Data+AllocOffs, Start, Len);
+  if (AllocOffs + Len <= AllocChunkSize) {
+    memcpy(AllocBuffer->Data + AllocOffs, Start, Len);
     AllocOffs += Len;
-    return RopePiece(AllocBuffer, AllocOffs-Len, AllocOffs);
+    return RopePiece(AllocBuffer, AllocOffs - Len, AllocOffs);
   }
 
   // If we don't have enough room because this specific allocation is huge,
   // just allocate a new rope piece for it alone.
   if (Len > AllocChunkSize) {
-    unsigned Size = End-Start+sizeof(RopeRefCountString)-1;
+    unsigned Size = End - Start + sizeof(RopeRefCountString) - 1;
     auto *Res = reinterpret_cast<RopeRefCountString *>(new char[Size]);
     Res->RefCount = 0;
-    memcpy(Res->Data, Start, End-Start);
-    return RopePiece(Res, 0, End-Start);
+    memcpy(Res->Data, Start, End - Start);
+    return RopePiece(Res, 0, End - Start);
   }
 
   // Otherwise, this was a small request but we just don't have space for it
diff --git a/llvm/lib/Support/SmallPtrSet.cpp b/llvm/lib/Support/SmallPtrSet.cpp
index cbb87ea8717cfc2..1bec911f39b13e8 100644
--- a/llvm/lib/Support/SmallPtrSet.cpp
+++ b/llvm/lib/Support/SmallPtrSet.cpp
@@ -62,7 +62,25 @@ SmallPtrSetImplBase::insert_imp_big(const void *Ptr) {
   return std::make_pair(Bucket, true);
 }
 
-const void * const *SmallPtrSetImplBase::FindBucketFor(const void *Ptr) const {
+const void *const *SmallPtrSetImplBase::doFind(const void *Ptr) const {
+  unsigned BucketNo =
+      DenseMapInfo<void *>::getHashValue(Ptr) & (CurArraySize - 1);
+  unsigned ProbeAmt = 1;
+  while (true) {
+    const void *const *Bucket = CurArray + BucketNo;
+    if (LLVM_LIKELY(*Bucket == Ptr))
+      return Bucket;
+    if (LLVM_LIKELY(*Bucket == getEmptyMarker()))
+      return nullptr;
+
+    // Otherwise, it's a hash collision or a tombstone, continue quadratic
+    // probing.
+    BucketNo += ProbeAmt++;
+    BucketNo &= CurArraySize - 1;
+  }
+}
+
+const void *const *SmallPtrSetImplBase::FindBucketFor(const void *Ptr) const {
   unsigned Bucket = DenseMapInfo<void *>::getHashValue(Ptr) & (CurArraySize-1);
   unsigned ArraySize = CurArraySize;
   unsigned ProbeAmt = 1;
diff --git a/llvm/lib/TableGen/Error.cpp b/llvm/lib/TableGen/Error.cpp
index dabb265ef80ca1f..4b99ff89da3b939 100644
--- a/llvm/lib/TableGen/Error.cpp
+++ b/llvm/lib/TableGen/Error.cpp
@@ -173,8 +173,11 @@ void CheckAssert(SMLoc Loc, Init *Condition, Init *Message) {
 // Dump a message to stderr.
 void dumpMessage(SMLoc Loc, Init *Message) {
   auto *MessageInit = dyn_cast<StringInit>(Message);
-  assert(MessageInit && "no debug message to print");
-  PrintNote(Loc, MessageInit->getValue());
+  if (!MessageInit) {
+    PrintError(Loc, "dump value is not of type string");
+  } else {
+    PrintNote(Loc, MessageInit->getValue());
+  }
 }
 
 } // end namespace llvm
diff --git a/llvm/lib/TableGen/TGLexer.cpp b/llvm/lib/TableGen/TGLexer.cpp
index 0554f0cf5788310..c1fae91e1fd9f2d 100644
--- a/llvm/lib/TableGen/TGLexer.cpp
+++ b/llvm/lib/TableGen/TGLexer.cpp
@@ -44,6 +44,25 @@ constexpr PreprocessorDir PreprocessorDirs[] = {{tgtok::Ifdef, "ifdef"},
                                                 {tgtok::Endif, "endif"},
                                                 {tgtok::Define, "define"}};
 
+// Returns a pointer past the end of a valid macro name at the start of `Str`.
+// Valid macro names match the regular expression [a-zA-Z_][0-9a-zA-Z_]*.
+static const char *lexMacroName(StringRef Str) {
+  assert(!Str.empty());
+
+  // Macro names start with [a-zA-Z_].
+  const char *Next = Str.begin();
+  if (*Next != '_' && !isalpha(*Next))
+    return Next;
+  // Eat the first character of the name.
+  ++Next;
+
+  // Match the rest of the identifier regex: [0-9a-zA-Z_]*
+  const char *End = Str.end();
+  while (Next != End && (isalpha(*Next) || isdigit(*Next) || *Next == '_'))
+    ++Next;
+  return Next;
+}
+
 TGLexer::TGLexer(SourceMgr &SM, ArrayRef<std::string> Macros) : SrcMgr(SM) {
   CurBuffer = SrcMgr.getMainFileID();
   CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
@@ -54,9 +73,16 @@ TGLexer::TGLexer(SourceMgr &SM, ArrayRef<std::string> Macros) : SrcMgr(SM) {
   PrepIncludeStack.push_back(
       std::make_unique<std::vector<PreprocessorControlDesc>>());
 
-  // Put all macros defined in the command line into the DefinedMacros set.
-  for (const std::string &MacroName : Macros)
+  // Add all macros defined on the command line to the DefinedMacros set.
+  // Check invalid macro names and print fatal error if we find one.
+  for (StringRef MacroName : Macros) {
+    const char *End = lexMacroName(MacroName);
+    if (End != MacroName.end())
+      PrintFatalError("Invalid macro name `" + MacroName +
+                      "` specified on command line");
+
     DefinedMacros.insert(MacroName);
+  }
 }
 
 SMLoc TGLexer::getLoc() const {
@@ -699,9 +725,8 @@ bool TGLexer::prepEatPreprocessorDirective(tgtok::TokKind Kind) {
   return false;
 }
 
-tgtok::TokKind TGLexer::lexPreprocessor(
-    tgtok::TokKind Kind, bool ReturnNextLiveToken) {
-
+tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
+                                        bool ReturnNextLiveToken) {
   // We must be looking at a preprocessing directive.  Eat it!
   if (!prepEatPreprocessorDirective(Kind))
     PrintFatalError("lexPreprocessor() called for unknown "
@@ -901,14 +926,7 @@ StringRef TGLexer::prepLexMacroName() {
     ++CurPtr;
 
   TokStart = CurPtr;
-  // Macro names start with [a-zA-Z_].
-  if (*CurPtr != '_' && !isalpha(*CurPtr))
-    return "";
-
-  // Match the rest of the identifier regex: [0-9a-zA-Z_]*
-  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
-    ++CurPtr;
-
+  CurPtr = lexMacroName(StringRef(CurPtr, CurBuf.end() - CurPtr));
   return StringRef(TokStart, CurPtr - TokStart);
 }
 
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index ce9da960545c231..d1d46105c814e96 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -4460,8 +4460,13 @@ bool TGParser::ParseDump(MultiClass *CurMultiClass, Record *CurRec) {
 
   if (CurRec)
     CurRec->addDump(Loc, Message);
-  else
-    addEntry(std::make_unique<Record::DumpInfo>(Loc, Message));
+  else {
+    HasReferenceResolver resolver{nullptr};
+    resolver.setFinal(true);
+    // force a resolution with a dummy resolver
+    Init *ResolvedMessage = Message->resolveReferences(resolver);
+    addEntry(std::make_unique<Record::DumpInfo>(Loc, ResolvedMessage));
+  }
 
   return false;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 8ec13f17cf0a0d4..69d6b02fefffe93 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -721,9 +721,6 @@ def FeatureTaggedGlobals : SubtargetFeature<"tagged-globals",
     "true", "Use an instruction sequence for taking the address of a global "
     "that allows a memory tag in the upper address bits">;
 
-def FeatureAppleA7SysReg  : SubtargetFeature<"apple-a7-sysreg", "HasAppleA7SysReg", "true",
-  "Apple A7 (the CPU formerly known as Cyclone)">;
-
 def FeatureEL2VMSA : SubtargetFeature<"el2vmsa", "HasEL2VMSA", "true",
   "Enable Exception Level 2 Virtual Memory System Architecture">;
 
@@ -782,27 +779,26 @@ def HasV8_2aOps : Architecture64<8, 2, "a", "v8.2a",
   [HasV8_1aOps, FeaturePsUAO, FeaturePAN_RWV, FeatureRAS, FeatureCCPP],
   !listconcat(HasV8_1aOps.DefaultExts, [FeatureRAS])>;
 def HasV8_3aOps : Architecture64<8, 3, "a", "v8.3a",
-  [HasV8_2aOps, FeatureRCPC, FeaturePAuth, FeatureJS, FeatureCCIDX,
-    FeatureComplxNum],
+  [HasV8_2aOps, FeatureRCPC, FeaturePAuth, FeatureJS, FeatureComplxNum],
   !listconcat(HasV8_2aOps.DefaultExts, [FeatureComplxNum, FeatureJS,
-    FeaturePAuth, FeatureRCPC])>;
+    FeaturePAuth, FeatureRCPC,  FeatureCCIDX])>;
 def HasV8_4aOps : Architecture64<8, 4, "a", "v8.4a",
   [HasV8_3aOps, FeatureDotProd, FeatureNV, FeatureMPAM, FeatureDIT,
     FeatureTRACEV8_4, FeatureAM, FeatureSEL2, FeatureTLB_RMI, FeatureFlagM,
     FeatureRCPC_IMMO, FeatureLSE2],
-  !listconcat(HasV8_3aOps.DefaultExts, [FeatureDotProd])>;
+  !listconcat(HasV8_3aOps.DefaultExts, [FeatureDotProd, FeatureDIT, FeatureFlagM])>;
 def HasV8_5aOps : Architecture64<8, 5, "a", "v8.5a",
   [HasV8_4aOps, FeatureAltFPCmp, FeatureFRInt3264, FeatureSpecRestrict,
-    FeatureSSBS, FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
+    FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
     FeatureBranchTargetId],
-  !listconcat(HasV8_4aOps.DefaultExts, [])>;
+  !listconcat(HasV8_4aOps.DefaultExts, [FeaturePredRes, FeatureSSBS, FeatureBranchTargetId, FeatureSB])>;
 def HasV8_6aOps : Architecture64<8, 6, "a", "v8.6a",
   [HasV8_5aOps, FeatureAMVS, FeatureBF16, FeatureFineGrainedTraps,
     FeatureEnhancedCounterVirtualization, FeatureMatMulInt8],
   !listconcat(HasV8_5aOps.DefaultExts, [FeatureBF16, FeatureMatMulInt8])>;
 def HasV8_7aOps : Architecture64<8, 7, "a", "v8.7a",
   [HasV8_6aOps, FeatureXS, FeatureWFxT, FeatureHCX],
-  !listconcat(HasV8_6aOps.DefaultExts, [])>;
+  !listconcat(HasV8_6aOps.DefaultExts, [FeatureWFxT])>;
 def HasV8_8aOps : Architecture64<8, 8, "a", "v8.8a",
   [HasV8_7aOps, FeatureHBC, FeatureMOPS, FeatureNMI],
   !listconcat(HasV8_7aOps.DefaultExts, [FeatureMOPS, FeatureHBC])>;
@@ -820,7 +816,7 @@ def HasV9_1aOps : Architecture64<9, 1, "a", "v9.1a",
   !listconcat(HasV9_0aOps.DefaultExts, [FeatureBF16, FeatureMatMulInt8, FeatureRME])>;
 def HasV9_2aOps : Architecture64<9, 2, "a", "v9.2a",
   [HasV8_7aOps, HasV9_1aOps],
-  !listconcat(HasV9_1aOps.DefaultExts, [FeatureMEC])>;
+  !listconcat(HasV9_1aOps.DefaultExts, [FeatureMEC, FeatureWFxT])>;
 def HasV9_3aOps : Architecture64<9, 3, "a", "v9.3a",
   [HasV8_8aOps, HasV9_2aOps],
   !listconcat(HasV9_2aOps.DefaultExts, [FeatureMOPS, FeatureHBC])>;
@@ -837,7 +833,7 @@ def HasV8_0rOps : Architecture64<8, 0, "r", "v8r",
     //v8.2
     FeatureRAS, FeaturePsUAO, FeatureCCPP, FeaturePAN_RWV,
     //v8.3
-    FeatureCCIDX, FeaturePAuth, FeatureRCPC,
+    FeaturePAuth, FeatureRCPC,
     //v8.4
     FeatureTRACEV8_4, FeatureTLB_RMI, FeatureFlagM, FeatureDIT, FeatureSEL2,
     FeatureRCPC_IMMO,
@@ -848,7 +844,7 @@ def HasV8_0rOps : Architecture64<8, 0, "r", "v8r",
   // For v8-R, we do not enable crypto and align with GCC that enables a more
   // minimal set of optional architecture extensions.
   !listconcat(
-    !listremove(HasV8_5aOps.DefaultExts, [FeatureLSE]),
+    !listremove(HasV8_5aOps.DefaultExts, [FeatureBranchTargetId, FeaturePredRes]),
     [FeatureSSBS, FeatureFullFP16, FeatureFP16FML, FeatureSB]
   )>;
 
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 429affed63497d5..e0f8853b715354b 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2860,7 +2860,8 @@ static bool produceCompactUnwindFrame(MachineFunction &MF) {
   return Subtarget.isTargetMachO() &&
          !(Subtarget.getTargetLowering()->supportSwiftError() &&
            Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
-         MF.getFunction().getCallingConv() != CallingConv::SwiftTail;
+         MF.getFunction().getCallingConv() != CallingConv::SwiftTail &&
+         !requiresSaveVG(MF);
 }
 
 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index d9b0f083d469cc3..f5f9a62faa0f534 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -913,7 +913,8 @@ bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
   if ((MulImm % std::abs(Scale)) == 0) {
     int64_t RDVLImm = MulImm / Scale;
     if ((RDVLImm >= Low) && (RDVLImm <= High)) {
-      Imm = CurDAG->getTargetConstant(RDVLImm, SDLoc(N), MVT::i32);
+      Imm = CurDAG->getSignedConstant(RDVLImm, SDLoc(N), MVT::i32,
+                                      /*isTarget=*/true);
       return true;
     }
   }
@@ -4245,7 +4246,7 @@ bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
     int64_t ImmVal = CNode->getSExtValue();
     SDLoc DL(N);
     if (ImmVal >= -128 && ImmVal < 128) {
-      Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
+      Imm = CurDAG->getSignedConstant(ImmVal, DL, MVT::i32, /*isTarget=*/true);
       return true;
     }
   }
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 97fb2c5f552731f..e1d265fdf0d1a8e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3970,9 +3970,9 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
         SDValue SExt =
             DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
                         DAG.getValueType(MVT::i16));
-        Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
-                                                   RHS.getValueType()),
-                             CC, dl, DAG);
+        Cmp = emitComparison(
+            SExt, DAG.getSignedConstant(ValueofRHS, dl, RHS.getValueType()), CC,
+            dl, DAG);
         AArch64CC = changeIntCCToAArch64CC(CC);
       }
     }
@@ -4158,7 +4158,7 @@ SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
 
     FVal = Other;
     TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
-                       DAG.getConstant(-1ULL, dl, Other.getValueType()));
+                       DAG.getAllOnesConstant(dl, Other.getValueType()));
 
     return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
                        CCVal, Cmp);
@@ -8966,7 +8966,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     // Each tail call may have to adjust the stack by a different amount, so
     // this information must travel along with the operation for eventual
     // consumption by emitEpilogue.
-    Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
+    Ops.push_back(
+        DAG.getSignedConstant(FPDiff, DL, MVT::i32, /*isTarget=*/true));
   }
 
   if (CLI.PAI) {
@@ -11144,7 +11145,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
 
     GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT);
     GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
-                        DAG.getConstant(GPRSize, DL, PtrVT));
+                        DAG.getSignedConstant(GPRSize, DL, PtrVT));
     GRTop = DAG.getZExtOrTrunc(GRTop, DL, PtrMemVT);
 
     MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
@@ -11162,7 +11163,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
 
     VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT);
     VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
-                        DAG.getConstant(FPRSize, DL, PtrVT));
+                        DAG.getSignedConstant(FPRSize, DL, PtrVT));
     VRTop = DAG.getZExtOrTrunc(VRTop, DL, PtrMemVT);
 
     MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
@@ -11175,7 +11176,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
   SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
                                    DAG.getConstant(Offset, DL, PtrVT));
   MemOps.push_back(
-      DAG.getStore(Chain, DL, DAG.getConstant(-GPRSize, DL, MVT::i32),
+      DAG.getStore(Chain, DL, DAG.getSignedConstant(-GPRSize, DL, MVT::i32),
                    GROffsAddr, MachinePointerInfo(SV, Offset), Align(4)));
 
   // int __vr_offs at offset 28 (16 on ILP32)
@@ -11183,7 +11184,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
   SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
                                    DAG.getConstant(Offset, DL, PtrVT));
   MemOps.push_back(
-      DAG.getStore(Chain, DL, DAG.getConstant(-FPRSize, DL, MVT::i32),
+      DAG.getStore(Chain, DL, DAG.getSignedConstant(-FPRSize, DL, MVT::i32),
                    VROffsAddr, MachinePointerInfo(SV, Offset), Align(4)));
 
   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
@@ -15620,7 +15621,7 @@ SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
   assert(VT != MVT::i128 && "Handled elsewhere, code replicated.");
   SDValue RHS = Op.getOperand(2);
   AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
-  RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS);
+  RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getAllOnesConstant(dl, VT), RHS);
   return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, dl, AN->getMemoryVT(),
                        Op.getOperand(0), Op.getOperand(1), RHS,
                        AN->getMemOperand());
@@ -21506,7 +21507,7 @@ static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
       if (auto *CN = dyn_cast<ConstantSDNode>(Comparator.getOperand(0))) {
         int64_t ImmVal = CN->getSExtValue();
         if (ImmVal >= -16 && ImmVal <= 15)
-          Imm = DAG.getConstant(ImmVal, DL, MVT::i32);
+          Imm = DAG.getSignedConstant(ImmVal, DL, MVT::i32);
         else
           return SDValue();
       }
@@ -24400,7 +24401,7 @@ static SDValue performSETCCCombine(SDNode *N,
       // this pattern will get better opt in emitComparison
       uint64_t TstImm = -1ULL << LHS->getConstantOperandVal(1);
       SDValue TST = DAG.getNode(ISD::AND, DL, TstVT, LHS->getOperand(0),
-                                DAG.getConstant(TstImm, DL, TstVT));
+                                DAG.getSignedConstant(TstImm, DL, TstVT));
       return DAG.getNode(ISD::SETCC, DL, VT, TST, RHS, N->getOperand(2));
     }
   }
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index e425984a6778443..697ae510a95655c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2344,132 +2344,258 @@ std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
 unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
   switch (Opc) {
   default:
+    llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
+  case AArch64::ADDG:
+  case AArch64::LDAPURBi:
+  case AArch64::LDAPURHi:
+  case AArch64::LDAPURi:
+  case AArch64::LDAPURSBWi:
+  case AArch64::LDAPURSBXi:
+  case AArch64::LDAPURSHWi:
+  case AArch64::LDAPURSHXi:
+  case AArch64::LDAPURSWi:
+  case AArch64::LDAPURXi:
+  case AArch64::LDR_PPXI:
+  case AArch64::LDR_PXI:
+  case AArch64::LDR_ZXI:
+  case AArch64::LDR_ZZXI:
+  case AArch64::LDR_ZZZXI:
+  case AArch64::LDR_ZZZZXI:
+  case AArch64::LDRBBui:
+  case AArch64::LDRBui:
+  case AArch64::LDRDui:
+  case AArch64::LDRHHui:
+  case AArch64::LDRHui:
+  case AArch64::LDRQui:
+  case AArch64::LDRSBWui:
+  case AArch64::LDRSBXui:
+  case AArch64::LDRSHWui:
+  case AArch64::LDRSHXui:
+  case AArch64::LDRSui:
+  case AArch64::LDRSWui:
+  case AArch64::LDRWui:
+  case AArch64::LDRXui:
+  case AArch64::LDURBBi:
+  case AArch64::LDURBi:
+  case AArch64::LDURDi:
+  case AArch64::LDURHHi:
+  case AArch64::LDURHi:
+  case AArch64::LDURQi:
+  case AArch64::LDURSBWi:
+  case AArch64::LDURSBXi:
+  case AArch64::LDURSHWi:
+  case AArch64::LDURSHXi:
+  case AArch64::LDURSi:
+  case AArch64::LDURSWi:
+  case AArch64::LDURWi:
+  case AArch64::LDURXi:
+  case AArch64::PRFMui:
+  case AArch64::PRFUMi:
+  case AArch64::ST2Gi:
+  case AArch64::STGi:
+  case AArch64::STLURBi:
+  case AArch64::STLURHi:
+  case AArch64::STLURWi:
+  case AArch64::STLURXi:
+  case AArch64::StoreSwiftAsyncContext:
+  case AArch64::STR_PPXI:
+  case AArch64::STR_PXI:
+  case AArch64::STR_ZXI:
+  case AArch64::STR_ZZXI:
+  case AArch64::STR_ZZZXI:
+  case AArch64::STR_ZZZZXI:
+  case AArch64::STRBBui:
+  case AArch64::STRBui:
+  case AArch64::STRDui:
+  case AArch64::STRHHui:
+  case AArch64::STRHui:
+  case AArch64::STRQui:
+  case AArch64::STRSui:
+  case AArch64::STRWui:
+  case AArch64::STRXui:
+  case AArch64::STURBBi:
+  case AArch64::STURBi:
+  case AArch64::STURDi:
+  case AArch64::STURHHi:
+  case AArch64::STURHi:
+  case AArch64::STURQi:
+  case AArch64::STURSi:
+  case AArch64::STURWi:
+  case AArch64::STURXi:
+  case AArch64::STZ2Gi:
+  case AArch64::STZGi:
+  case AArch64::TAGPstack:
     return 2;
-  case AArch64::LDPXi:
-  case AArch64::LDPDi:
-  case AArch64::STPXi:
-  case AArch64::STPDi:
-  case AArch64::LDNPXi:
-  case AArch64::LDNPDi:
-  case AArch64::STNPXi:
-  case AArch64::STNPDi:
-  case AArch64::LDPQi:
-  case AArch64::STPQi:
-  case AArch64::LDNPQi:
-  case AArch64::STNPQi:
-  case AArch64::LDPWi:
-  case AArch64::LDPSi:
-  case AArch64::STPWi:
-  case AArch64::STPSi:
-  case AArch64::LDNPWi:
-  case AArch64::LDNPSi:
-  case AArch64::STNPWi:
-  case AArch64::STNPSi:
-  case AArch64::LDG:
-  case AArch64::STGPi:
-
-  case AArch64::LD1B_IMM:
+  case AArch64::LD1B_D_IMM:
   case AArch64::LD1B_H_IMM:
+  case AArch64::LD1B_IMM:
   case AArch64::LD1B_S_IMM:
-  case AArch64::LD1B_D_IMM:
-  case AArch64::LD1SB_H_IMM:
-  case AArch64::LD1SB_S_IMM:
-  case AArch64::LD1SB_D_IMM:
+  case AArch64::LD1D_IMM:
+  case AArch64::LD1H_D_IMM:
   case AArch64::LD1H_IMM:
   case AArch64::LD1H_S_IMM:
-  case AArch64::LD1H_D_IMM:
-  case AArch64::LD1SH_S_IMM:
+  case AArch64::LD1RB_D_IMM:
+  case AArch64::LD1RB_H_IMM:
+  case AArch64::LD1RB_IMM:
+  case AArch64::LD1RB_S_IMM:
+  case AArch64::LD1RD_IMM:
+  case AArch64::LD1RH_D_IMM:
+  case AArch64::LD1RH_IMM:
+  case AArch64::LD1RH_S_IMM:
+  case AArch64::LD1RSB_D_IMM:
+  case AArch64::LD1RSB_H_IMM:
+  case AArch64::LD1RSB_S_IMM:
+  case AArch64::LD1RSH_D_IMM:
+  case AArch64::LD1RSH_S_IMM:
+  case AArch64::LD1RSW_IMM:
+  case AArch64::LD1RW_D_IMM:
+  case AArch64::LD1RW_IMM:
+  case AArch64::LD1SB_D_IMM:
+  case AArch64::LD1SB_H_IMM:
+  case AArch64::LD1SB_S_IMM:
   case AArch64::LD1SH_D_IMM:
-  case AArch64::LD1W_IMM:
-  case AArch64::LD1W_D_IMM:
+  case AArch64::LD1SH_S_IMM:
   case AArch64::LD1SW_D_IMM:
-  case AArch64::LD1D_IMM:
-
+  case AArch64::LD1W_D_IMM:
+  case AArch64::LD1W_IMM:
   case AArch64::LD2B_IMM:
+  case AArch64::LD2D_IMM:
   case AArch64::LD2H_IMM:
   case AArch64::LD2W_IMM:
-  case AArch64::LD2D_IMM:
   case AArch64::LD3B_IMM:
+  case AArch64::LD3D_IMM:
   case AArch64::LD3H_IMM:
   case AArch64::LD3W_IMM:
-  case AArch64::LD3D_IMM:
   case AArch64::LD4B_IMM:
+  case AArch64::LD4D_IMM:
   case AArch64::LD4H_IMM:
   case AArch64::LD4W_IMM:
-  case AArch64::LD4D_IMM:
-
-  case AArch64::ST1B_IMM:
+  case AArch64::LDG:
+  case AArch64::LDNF1B_D_IMM:
+  case AArch64::LDNF1B_H_IMM:
+  case AArch64::LDNF1B_IMM:
+  case AArch64::LDNF1B_S_IMM:
+  case AArch64::LDNF1D_IMM:
+  case AArch64::LDNF1H_D_IMM:
+  case AArch64::LDNF1H_IMM:
+  case AArch64::LDNF1H_S_IMM:
+  case AArch64::LDNF1SB_D_IMM:
+  case AArch64::LDNF1SB_H_IMM:
+  case AArch64::LDNF1SB_S_IMM:
+  case AArch64::LDNF1SH_D_IMM:
+  case AArch64::LDNF1SH_S_IMM:
+  case AArch64::LDNF1SW_D_IMM:
+  case AArch64::LDNF1W_D_IMM:
+  case AArch64::LDNF1W_IMM:
+  case AArch64::LDNPDi:
+  case AArch64::LDNPQi:
+  case AArch64::LDNPSi:
+  case AArch64::LDNPWi:
+  case AArch64::LDNPXi:
+  case AArch64::LDNT1B_ZRI:
+  case AArch64::LDNT1D_ZRI:
+  case AArch64::LDNT1H_ZRI:
+  case AArch64::LDNT1W_ZRI:
+  case AArch64::LDPDi:
+  case AArch64::LDPQi:
+  case AArch64::LDPSi:
+  case AArch64::LDPWi:
+  case AArch64::LDPXi:
+  case AArch64::LDRBBpost:
+  case AArch64::LDRBBpre:
+  case AArch64::LDRBpost:
+  case AArch64::LDRBpre:
+  case AArch64::LDRDpost:
+  case AArch64::LDRDpre:
+  case AArch64::LDRHHpost:
+  case AArch64::LDRHHpre:
+  case AArch64::LDRHpost:
+  case AArch64::LDRHpre:
+  case AArch64::LDRQpost:
+  case AArch64::LDRQpre:
+  case AArch64::LDRSpost:
+  case AArch64::LDRSpre:
+  case AArch64::LDRWpost:
+  case AArch64::LDRWpre:
+  case AArch64::LDRXpost:
+  case AArch64::LDRXpre:
+  case AArch64::ST1B_D_IMM:
   case AArch64::ST1B_H_IMM:
+  case AArch64::ST1B_IMM:
   case AArch64::ST1B_S_IMM:
-  case AArch64::ST1B_D_IMM:
+  case AArch64::ST1D_IMM:
+  case AArch64::ST1H_D_IMM:
   case AArch64::ST1H_IMM:
   case AArch64::ST1H_S_IMM:
-  case AArch64::ST1H_D_IMM:
-  case AArch64::ST1W_IMM:
   case AArch64::ST1W_D_IMM:
-  case AArch64::ST1D_IMM:
-
+  case AArch64::ST1W_IMM:
   case AArch64::ST2B_IMM:
+  case AArch64::ST2D_IMM:
   case AArch64::ST2H_IMM:
   case AArch64::ST2W_IMM:
-  case AArch64::ST2D_IMM:
   case AArch64::ST3B_IMM:
+  case AArch64::ST3D_IMM:
   case AArch64::ST3H_IMM:
   case AArch64::ST3W_IMM:
-  case AArch64::ST3D_IMM:
   case AArch64::ST4B_IMM:
+  case AArch64::ST4D_IMM:
   case AArch64::ST4H_IMM:
   case AArch64::ST4W_IMM:
-  case AArch64::ST4D_IMM:
-
-  case AArch64::LD1RB_IMM:
-  case AArch64::LD1RB_H_IMM:
-  case AArch64::LD1RB_S_IMM:
-  case AArch64::LD1RB_D_IMM:
-  case AArch64::LD1RSB_H_IMM:
-  case AArch64::LD1RSB_S_IMM:
-  case AArch64::LD1RSB_D_IMM:
-  case AArch64::LD1RH_IMM:
-  case AArch64::LD1RH_S_IMM:
-  case AArch64::LD1RH_D_IMM:
-  case AArch64::LD1RSH_S_IMM:
-  case AArch64::LD1RSH_D_IMM:
-  case AArch64::LD1RW_IMM:
-  case AArch64::LD1RW_D_IMM:
-  case AArch64::LD1RSW_IMM:
-  case AArch64::LD1RD_IMM:
-
-  case AArch64::LDNT1B_ZRI:
-  case AArch64::LDNT1H_ZRI:
-  case AArch64::LDNT1W_ZRI:
-  case AArch64::LDNT1D_ZRI:
+  case AArch64::STGPi:
+  case AArch64::STNPDi:
+  case AArch64::STNPQi:
+  case AArch64::STNPSi:
+  case AArch64::STNPWi:
+  case AArch64::STNPXi:
   case AArch64::STNT1B_ZRI:
+  case AArch64::STNT1D_ZRI:
   case AArch64::STNT1H_ZRI:
   case AArch64::STNT1W_ZRI:
-  case AArch64::STNT1D_ZRI:
-
-  case AArch64::LDNF1B_IMM:
-  case AArch64::LDNF1B_H_IMM:
-  case AArch64::LDNF1B_S_IMM:
-  case AArch64::LDNF1B_D_IMM:
-  case AArch64::LDNF1SB_H_IMM:
-  case AArch64::LDNF1SB_S_IMM:
-  case AArch64::LDNF1SB_D_IMM:
-  case AArch64::LDNF1H_IMM:
-  case AArch64::LDNF1H_S_IMM:
-  case AArch64::LDNF1H_D_IMM:
-  case AArch64::LDNF1SH_S_IMM:
-  case AArch64::LDNF1SH_D_IMM:
-  case AArch64::LDNF1W_IMM:
-  case AArch64::LDNF1W_D_IMM:
-  case AArch64::LDNF1SW_D_IMM:
-  case AArch64::LDNF1D_IMM:
+  case AArch64::STPDi:
+  case AArch64::STPQi:
+  case AArch64::STPSi:
+  case AArch64::STPWi:
+  case AArch64::STPXi:
+  case AArch64::STRBBpost:
+  case AArch64::STRBBpre:
+  case AArch64::STRBpost:
+  case AArch64::STRBpre:
+  case AArch64::STRDpost:
+  case AArch64::STRDpre:
+  case AArch64::STRHHpost:
+  case AArch64::STRHHpre:
+  case AArch64::STRHpost:
+  case AArch64::STRHpre:
+  case AArch64::STRQpost:
+  case AArch64::STRQpre:
+  case AArch64::STRSpost:
+  case AArch64::STRSpre:
+  case AArch64::STRWpost:
+  case AArch64::STRWpre:
+  case AArch64::STRXpost:
+  case AArch64::STRXpre:
     return 3;
-  case AArch64::ADDG:
-  case AArch64::STGi:
-  case AArch64::LDR_PXI:
-  case AArch64::STR_PXI:
-    return 2;
+  case AArch64::LDPDpost:
+  case AArch64::LDPDpre:
+  case AArch64::LDPQpost:
+  case AArch64::LDPQpre:
+  case AArch64::LDPSpost:
+  case AArch64::LDPSpre:
+  case AArch64::LDPWpost:
+  case AArch64::LDPWpre:
+  case AArch64::LDPXpost:
+  case AArch64::LDPXpre:
+  case AArch64::STPDpost:
+  case AArch64::STPDpre:
+  case AArch64::STPQpost:
+  case AArch64::STPQpre:
+  case AArch64::STPSpost:
+  case AArch64::STPSpre:
+  case AArch64::STPWpost:
+  case AArch64::STPWpre:
+  case AArch64::STPXpost:
+  case AArch64::STPXpre:
+    return 4;
   }
 }
 
@@ -3834,22 +3960,58 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
     MinOffset = -256;
     MaxOffset = 255;
     break;
-  case AArch64::STRXpre:
-  case AArch64::STRDpre:
-  case AArch64::LDRXpost:
   case AArch64::LDRDpost:
+  case AArch64::LDRDpre:
+  case AArch64::LDRXpost:
+  case AArch64::LDRXpre:
+  case AArch64::STRDpost:
+  case AArch64::STRDpre:
+  case AArch64::STRXpost:
+  case AArch64::STRXpre:
     Scale = TypeSize::getFixed(1);
     Width = TypeSize::getFixed(8);
     MinOffset = -256;
     MaxOffset = 255;
     break;
   case AArch64::STRWpost:
+  case AArch64::STRWpre:
   case AArch64::LDRWpost:
+  case AArch64::LDRWpre:
+  case AArch64::STRSpost:
+  case AArch64::STRSpre:
+  case AArch64::LDRSpost:
+  case AArch64::LDRSpre:
     Scale = TypeSize::getFixed(1);
     Width = TypeSize::getFixed(4);
     MinOffset = -256;
     MaxOffset = 255;
     break;
+  case AArch64::LDRHpost:
+  case AArch64::LDRHpre:
+  case AArch64::STRHpost:
+  case AArch64::STRHpre:
+  case AArch64::LDRHHpost:
+  case AArch64::LDRHHpre:
+  case AArch64::STRHHpost:
+  case AArch64::STRHHpre:
+    Scale = TypeSize::getFixed(1);
+    Width = TypeSize::getFixed(2);
+    MinOffset = -256;
+    MaxOffset = 255;
+    break;
+  case AArch64::LDRBpost:
+  case AArch64::LDRBpre:
+  case AArch64::STRBpost:
+  case AArch64::STRBpre:
+  case AArch64::LDRBBpost:
+  case AArch64::LDRBBpre:
+  case AArch64::STRBBpost:
+  case AArch64::STRBBpre:
+    Scale = TypeSize::getFixed(1);
+    Width = TypeSize::getFixed(1);
+    MinOffset = -256;
+    MaxOffset = 255;
+    break;
   // Unscaled
   case AArch64::LDURQi:
   case AArch64::STURQi:
@@ -3918,8 +4080,10 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
   case AArch64::LDNPQi:
   case AArch64::STPQi:
   case AArch64::STNPQi:
-  case AArch64::STPQpre:
   case AArch64::LDPQpost:
+  case AArch64::LDPQpre:
+  case AArch64::STPQpost:
+  case AArch64::STPQpre:
     Scale = TypeSize::getFixed(16);
     Width = TypeSize::getFixed(16 * 2);
     MinOffset = -64;
@@ -3933,10 +4097,14 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
   case AArch64::STPDi:
   case AArch64::STNPXi:
   case AArch64::STNPDi:
-  case AArch64::STPXpre:
+  case AArch64::LDPDpost:
+  case AArch64::LDPDpre:
   case AArch64::LDPXpost:
+  case AArch64::LDPXpre:
+  case AArch64::STPDpost:
   case AArch64::STPDpre:
-  case AArch64::LDPDpost:
+  case AArch64::STPXpost:
+  case AArch64::STPXpre:
     Scale = TypeSize::getFixed(8);
     Width = TypeSize::getFixed(8 * 2);
     MinOffset = -64;
@@ -3950,6 +4118,14 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
   case AArch64::STPSi:
   case AArch64::STNPWi:
   case AArch64::STNPSi:
+  case AArch64::LDPSpost:
+  case AArch64::LDPSpre:
+  case AArch64::LDPWpost:
+  case AArch64::LDPWpre:
+  case AArch64::STPSpost:
+  case AArch64::STPSpre:
+  case AArch64::STPWpost:
+  case AArch64::STPWpre:
     Scale = TypeSize::getFixed(4);
     Width = TypeSize::getFixed(4 * 2);
     MinOffset = -64;
@@ -4340,6 +4516,16 @@ AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
   return MI.getOperand(Idx);
 }
 
+const MachineOperand &
+AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
+  switch (MI.getOpcode()) {
+  default:
+    llvm_unreachable("Unexpected opcode");
+  case AArch64::LDRBBroX:
+    return MI.getOperand(4);
+  }
+}
+
 static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
                                               Register Reg) {
   if (MI.getParent() == nullptr)
@@ -8875,8 +9061,8 @@ AArch64InstrInfo::getOutliningCandidateInfo(
     // link register.
     outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
     bool ModStackToSaveLR = false;
-    if (std::any_of(FirstCand.begin(), std::prev(FirstCand.end()),
-                    [](const MachineInstr &MI) { return MI.isCall(); }))
+    if (any_of(drop_end(FirstCand),
+               [](const MachineInstr &MI) { return MI.isCall(); }))
       ModStackToSaveLR = true;
 
     // Handle the last instruction separately. If this is a tail call, then the
@@ -10250,6 +10436,28 @@ AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
       Init, IsUpdatePriorComp, Cond);
 }
 
+/// verifyInstruction - Perform target specific instruction verification.
+bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI,
+                                         StringRef &ErrInfo) const {
+
+  // Verify that immediate offsets on load/store instructions are within range.
+  // Stack objects with an FI operand are excluded as they can be fixed up
+  // during PEI.
+  TypeSize Scale(0U, false), Width(0U, false);
+  int64_t MinOffset, MaxOffset;
+  if (getMemOpInfo(MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
+    unsigned ImmIdx = getLoadStoreImmIdx(MI.getOpcode());
+    if (MI.getOperand(ImmIdx).isImm() && !MI.getOperand(ImmIdx - 1).isFI()) {
+      int64_t Imm = MI.getOperand(ImmIdx).getImm();
+      if (Imm < MinOffset || Imm > MaxOffset) {
+        ErrInfo = "Unexpected immediate on load/store instruction";
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
 #define GET_INSTRINFO_HELPERS
 #define GET_INSTRMAP_INFO
 #include "AArch64GenInstrInfo.inc"
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 24a500aa190abbc..a1f2fbff016312a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -254,6 +254,9 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   /// Returns whether the physical register is FP or NEON.
   static bool isFpOrNEON(Register Reg);
 
+  /// Returns the shift amount operator of a load/store.
+  static const MachineOperand &getLdStAmountOp(const MachineInstr &MI);
+
   /// Returns whether the instruction is FP or NEON.
   static bool isFpOrNEON(const MachineInstr &MI);
 
@@ -583,6 +586,10 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   std::optional<unsigned>
   canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
                       MachineInstr *Pred, const MachineRegisterInfo *MRI) const;
+
+  /// verifyInstruction - Perform target specific instruction verification.
+  bool verifyInstruction(const MachineInstr &MI,
+                         StringRef &ErrInfo) const override;
 };
 
 struct UsedNZCV {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 403453a4ee0a92c..a5fae8029deab35 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -2214,7 +2214,7 @@ def s64imm_32bit : ImmLeaf<i64, [{
 }]>;
 
 def trunc_imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i32);
+  return CurDAG->getTargetConstant((uint32_t)N->getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 def gi_trunc_imm : GICustomOperandRenderer<"renderTruncImm">,
@@ -10148,7 +10148,7 @@ let Predicates = [HasD128] in {
 // 2023 Architecture Extensions:
 //===----------------------------===//
 
-let Predicates = [HasFP8] in {
+let Uses = [FPMR, FPCR], Predicates = [HasFP8] in {
   defm F1CVTL  : SIMDMixedTwoVectorFP8<0b00, "f1cvtl">;
   defm F2CVTL  : SIMDMixedTwoVectorFP8<0b01, "f2cvtl">;
   defm BF1CVTL : SIMDMixedTwoVectorFP8<0b10, "bf1cvtl">;
@@ -10173,7 +10173,7 @@ let Predicates = [HasNEON, HasFAMINMAX] in {
  }
 } // End let Predicates = [HasNEON, HasFAMINMAX]
 
-let Predicates = [HasFP8FMA] in {
+let Uses = [FPMR, FPCR], Predicates = [HasFP8FMA] in {
  defm FMLALBlane : SIMDThreeSameVectorMLAIndex<0b0, "fmlalb">;
  defm FMLALTlane : SIMDThreeSameVectorMLAIndex<0b1, "fmlalt">;
  defm FMLALLBBlane : SIMDThreeSameVectorMLALIndex<0b0, 0b00, "fmlallbb">;
@@ -10189,12 +10189,12 @@ let Predicates = [HasFP8FMA] in {
  defm FMLALLTT : SIMDThreeSameVectorMLAL<0b1, 0b01, "fmlalltt">;
 } // End let Predicates = [HasFP8FMA]
 
-let Predicates = [HasFP8DOT2] in {
+let Uses = [FPMR, FPCR], Predicates = [HasFP8DOT2] in {
  defm FDOTlane : SIMDThreeSameVectorFP8DOT2Index<"fdot">;
  defm FDOT : SIMDThreeSameVectorDOT2<"fdot">;
 } // End let Predicates = [HasFP8DOT2]
 
-let Predicates = [HasFP8DOT4] in {
+let Uses = [FPMR, FPCR], Predicates = [HasFP8DOT4] in {
  defm FDOTlane : SIMDThreeSameVectorFP8DOT4Index<"fdot">;
  defm FDOT : SIMDThreeSameVectorDOT4<"fdot">;
 } // End let Predicates = [HasFP8DOT4]
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index ff3a21cd0da3b7c..8de3f8db84ae2b7 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -64,6 +64,8 @@ STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
 STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
 STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation "
                                    "not passed the alignment check");
+STATISTIC(NumConstOffsetFolded,
+          "Number of const offset of index address folded");
 
 DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming",
               "Controls which pairs are considered for renaming");
@@ -77,6 +79,11 @@ static cl::opt<unsigned> LdStLimit("aarch64-load-store-scan-limit",
 static cl::opt<unsigned> UpdateLimit("aarch64-update-scan-limit", cl::init(100),
                                      cl::Hidden);
 
+// The LdStConstLimit limits how far we search for const offset instructions
+// when we form index address load/store instructions.
+static cl::opt<unsigned> LdStConstLimit("aarch64-load-store-const-scan-limit",
+                                        cl::init(10), cl::Hidden);
+
 // Enable register renaming to find additional store pairing opportunities.
 static cl::opt<bool> EnableRenaming("aarch64-load-store-renaming",
                                     cl::init(true), cl::Hidden);
@@ -173,6 +180,13 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
   findMatchingUpdateInsnForward(MachineBasicBlock::iterator I,
                                 int UnscaledOffset, unsigned Limit);
 
+  // Scan the instruction list to find a register assigned with a const
+  // value that can be combined with the current instruction (a load or store)
+  // using base addressing with writeback. Scan backwards.
+  MachineBasicBlock::iterator
+  findMatchingConstOffsetBackward(MachineBasicBlock::iterator I, unsigned Limit,
+                                  unsigned &Offset);
+
   // Scan the instruction list to find a base register update that can
   // be combined with the current instruction (a load or store) using
   // pre or post indexed addressing with writeback. Scan backwards.
@@ -184,11 +198,19 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
   bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI,
                             unsigned BaseReg, int Offset);
 
+  bool isMatchingMovConstInsn(MachineInstr &MemMI, MachineInstr &MI,
+                              unsigned IndexReg, unsigned &Offset);
+
   // Merge a pre- or post-index base register update into a ld/st instruction.
   MachineBasicBlock::iterator
   mergeUpdateInsn(MachineBasicBlock::iterator I,
                   MachineBasicBlock::iterator Update, bool IsPreIdx);
 
+  MachineBasicBlock::iterator
+  mergeConstOffsetInsn(MachineBasicBlock::iterator I,
+                       MachineBasicBlock::iterator Update, unsigned Offset,
+                       int Scale);
+
   // Find and merge zero store instructions.
   bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
 
@@ -201,6 +223,9 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
   // Find and merge a base register updates before or after a ld/st instruction.
   bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI);
 
+  // Find and merge an index ldr/st instruction into a base ld/st instruction.
+  bool tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI, int Scale);
+
   bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
 
   bool runOnMachineFunction(MachineFunction &Fn) override;
@@ -483,6 +508,16 @@ static unsigned getPreIndexedOpcode(unsigned Opc) {
   }
 }
 
+static unsigned getBaseAddressOpcode(unsigned Opc) {
+  // TODO: Add more index address loads/stores.
+  switch (Opc) {
+  default:
+    llvm_unreachable("Opcode has no base address equivalent!");
+  case AArch64::LDRBBroX:
+    return AArch64::LDRBBui;
+  }
+}
+
 static unsigned getPostIndexedOpcode(unsigned Opc) {
   switch (Opc) {
   default:
@@ -724,6 +759,20 @@ static bool isMergeableLdStUpdate(MachineInstr &MI) {
   }
 }
 
+// Make sure this is a reg+reg Ld/St
+static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) {
+  unsigned Opc = MI.getOpcode();
+  switch (Opc) {
+  default:
+    return false;
+  // Scaled instructions.
+  // TODO: Add more index address loads/stores.
+  case AArch64::LDRBBroX:
+    Scale = 1;
+    return true;
+  }
+}
+
 static bool isRewritableImplicitDef(unsigned Opc) {
   switch (Opc) {
   default:
@@ -2053,6 +2102,63 @@ AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I,
   return NextI;
 }
 
+MachineBasicBlock::iterator
+AArch64LoadStoreOpt::mergeConstOffsetInsn(MachineBasicBlock::iterator I,
+                                          MachineBasicBlock::iterator Update,
+                                          unsigned Offset, int Scale) {
+  assert((Update->getOpcode() == AArch64::MOVKWi) &&
+         "Unexpected const mov instruction to merge!");
+  MachineBasicBlock::iterator E = I->getParent()->end();
+  MachineBasicBlock::iterator NextI = next_nodbg(I, E);
+  MachineBasicBlock::iterator PrevI = prev_nodbg(Update, E);
+  MachineInstr &MemMI = *I;
+  unsigned Mask = (1 << 12) * Scale - 1;
+  unsigned Low = Offset & Mask;
+  unsigned High = Offset - Low;
+  Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MemMI).getReg();
+  Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MemMI).getReg();
+  MachineInstrBuilder AddMIB, MemMIB;
+
+  // Add IndexReg, BaseReg, High (the BaseReg may be SP)
+  AddMIB =
+      BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(AArch64::ADDXri))
+          .addDef(IndexReg)
+          .addUse(BaseReg)
+          .addImm(High >> 12) // shifted value
+          .addImm(12);        // shift 12
+  (void)AddMIB;
+  // Ld/St DestReg, IndexReg, Imm12
+  unsigned NewOpc = getBaseAddressOpcode(I->getOpcode());
+  MemMIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
+               .add(getLdStRegOp(MemMI))
+               .add(AArch64InstrInfo::getLdStOffsetOp(MemMI))
+               .addImm(Low / Scale)
+               .setMemRefs(I->memoperands())
+               .setMIFlags(I->mergeFlagsWith(*Update));
+  (void)MemMIB;
+
+  ++NumConstOffsetFolded;
+  LLVM_DEBUG(dbgs() << "Creating base address load/store.\n");
+  LLVM_DEBUG(dbgs() << "    Replacing instructions:\n    ");
+  LLVM_DEBUG(PrevI->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "    ");
+  LLVM_DEBUG(Update->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "    ");
+  LLVM_DEBUG(I->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "  with instruction:\n    ");
+  LLVM_DEBUG(((MachineInstr *)AddMIB)->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "    ");
+  LLVM_DEBUG(((MachineInstr *)MemMIB)->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "\n");
+
+  // Erase the old instructions for the block.
+  I->eraseFromParent();
+  PrevI->eraseFromParent();
+  Update->eraseFromParent();
+
+  return NextI;
+}
+
 bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
                                                MachineInstr &MI,
                                                unsigned BaseReg, int Offset) {
@@ -2100,6 +2206,34 @@ bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
   return false;
 }
 
+bool AArch64LoadStoreOpt::isMatchingMovConstInsn(MachineInstr &MemMI,
+                                                 MachineInstr &MI,
+                                                 unsigned IndexReg,
+                                                 unsigned &Offset) {
+  // The update instruction source and destination register must be the
+  // same as the load/store index register.
+  if (MI.getOpcode() == AArch64::MOVKWi &&
+      TRI->isSuperOrSubRegisterEq(IndexReg, MI.getOperand(1).getReg())) {
+
+    // movz + movk hold a large offset of a Ld/St instruction.
+    MachineBasicBlock::iterator B = MI.getParent()->begin();
+    MachineBasicBlock::iterator MBBI = &MI;
+    // Skip the scene when the MI is the first instruction of a block.
+    if (MBBI == B)
+      return false;
+    MBBI = prev_nodbg(MBBI, B);
+    MachineInstr &MovzMI = *MBBI;
+    if (MovzMI.getOpcode() == AArch64::MOVZWi) {
+      unsigned Low = MovzMI.getOperand(1).getImm();
+      unsigned High = MI.getOperand(2).getImm() << MI.getOperand(3).getImm();
+      Offset = High + Low;
+      // 12-bit optionally shifted immediates are legal for adds.
+      return Offset >> 24 == 0;
+    }
+  }
+  return false;
+}
+
 MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
     MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
   MachineBasicBlock::iterator E = I->getParent()->end();
@@ -2255,6 +2389,60 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
   return E;
 }
 
+MachineBasicBlock::iterator
+AArch64LoadStoreOpt::findMatchingConstOffsetBackward(
+    MachineBasicBlock::iterator I, unsigned Limit, unsigned &Offset) {
+  MachineBasicBlock::iterator B = I->getParent()->begin();
+  MachineBasicBlock::iterator E = I->getParent()->end();
+  MachineInstr &MemMI = *I;
+  MachineBasicBlock::iterator MBBI = I;
+
+  // If the load is the first instruction in the block, there's obviously
+  // not any matching load or store.
+  if (MBBI == B)
+    return E;
+
+  // Make sure the IndexReg is killed and the shift amount is zero.
+  // TODO: Relex this restriction to extend, simplify processing now.
+  if (!AArch64InstrInfo::getLdStOffsetOp(MemMI).isKill() ||
+      !AArch64InstrInfo::getLdStAmountOp(MemMI).isImm() ||
+      (AArch64InstrInfo::getLdStAmountOp(MemMI).getImm() != 0))
+    return E;
+
+  Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MemMI).getReg();
+
+  // Track which register units have been modified and used between the first
+  // insn (inclusive) and the second insn.
+  ModifiedRegUnits.clear();
+  UsedRegUnits.clear();
+  unsigned Count = 0;
+  do {
+    MBBI = prev_nodbg(MBBI, B);
+    MachineInstr &MI = *MBBI;
+
+    // Don't count transient instructions towards the search limit since there
+    // may be different numbers of them if e.g. debug information is present.
+    if (!MI.isTransient())
+      ++Count;
+
+    // If we found a match, return it.
+    if (isMatchingMovConstInsn(*I, MI, IndexReg, Offset)) {
+      return MBBI;
+    }
+
+    // Update the status of what the instruction clobbered and used.
+    LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
+
+    // Otherwise, if the index register is used or modified, we have no match,
+    // so return early.
+    if (!ModifiedRegUnits.available(IndexReg) ||
+        !UsedRegUnits.available(IndexReg))
+      return E;
+
+  } while (MBBI != B && Count < Limit);
+  return E;
+}
+
 bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
     MachineBasicBlock::iterator &MBBI) {
   MachineInstr &MI = *MBBI;
@@ -2443,6 +2631,34 @@ bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
   return false;
 }
 
+bool AArch64LoadStoreOpt::tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI,
+                                              int Scale) {
+  MachineInstr &MI = *MBBI;
+  MachineBasicBlock::iterator E = MI.getParent()->end();
+  MachineBasicBlock::iterator Update;
+
+  // Don't know how to handle unscaled pre/post-index versions below, so bail.
+  if (TII->hasUnscaledLdStOffset(MI.getOpcode()))
+    return false;
+
+  // Look back to try to find a const offset for index LdSt instruction. For
+  // example,
+  // mov x8, #LargeImm   ; = a * (1<<12) + imm12
+  // ldr x1, [x0, x8]
+  // merged into:
+  // add x8, x0, a * (1<<12)
+  // ldr x1, [x8, imm12]
+  unsigned Offset;
+  Update = findMatchingConstOffsetBackward(MBBI, LdStConstLimit, Offset);
+  if (Update != E && (Offset & (Scale - 1)) == 0) {
+    // Merge the imm12 into the ld/st.
+    MBBI = mergeConstOffsetInsn(MBBI, Update, Offset, Scale);
+    return true;
+  }
+
+  return false;
+}
+
 bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
                                         bool EnableNarrowZeroStOpt) {
 
@@ -2521,6 +2737,22 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
       ++MBBI;
   }
 
+  // 5) Find a register assigned with a const value that can be combined with
+  // into the load or store. e.g.,
+  //        mov x8, #LargeImm   ; = a * (1<<12) + imm12
+  //        ldr x1, [x0, x8]
+  //        ; becomes
+  //        add x8, x0, a * (1<<12)
+  //        ldr x1, [x8, imm12]
+  for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+       MBBI != E;) {
+    int Scale;
+    if (isMergeableIndexLdSt(*MBBI, Scale) && tryToMergeIndexLdSt(MBBI, Scale))
+      Modified = true;
+    else
+      ++MBBI;
+  }
+
   return Modified;
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 52b5c8a0903ea67..84d8cae3a0a5d18 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -688,6 +688,7 @@ def ProcessorFeatures {
                                  FeatureMatMulInt8, FeatureBF16, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureSB, FeaturePAuth, FeatureSSBS, FeatureSVE, FeatureSVE2,
                                  FeatureComplxNum, FeatureCRC, FeatureDotProd,
                                  FeatureFPARMv8,FeatureFullFP16, FeatureJS, FeatureLSE,
@@ -695,6 +696,7 @@ def ProcessorFeatures {
   list<SubtargetFeature> A520 = [HasV9_2aOps, FeaturePerfMon, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
                                  FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                  FeatureFPARMv8, FeatureFullFP16, FeatureMatMulInt8, FeatureJS,
@@ -703,6 +705,7 @@ def ProcessorFeatures {
   list<SubtargetFeature> A520AE = [HasV9_2aOps, FeaturePerfMon, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
                                  FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                  FeatureFPARMv8, FeatureFullFP16, FeatureMatMulInt8, FeatureJS,
@@ -734,12 +737,14 @@ def ProcessorFeatures {
                                  FeaturePerfMon, FeatureRCPC, FeatureSPE,
                                  FeatureSSBS, FeatureCRC, FeatureLSE, FeatureRAS, FeatureRDM];
   list<SubtargetFeature> A710 = [HasV9_0aOps, FeatureNEON, FeaturePerfMon,
+                                 FeatureCCIDX, FeatureSSBS,
                                  FeatureETE, FeatureMTE, FeatureFP16FML,
                                  FeatureSVE2BitPerm, FeatureBF16, FeatureMatMulInt8,
                                  FeaturePAuth, FeatureFlagM, FeatureSB, FeatureSVE, FeatureSVE2,
                                  FeatureComplxNum, FeatureCRC, FeatureDotProd, FeatureFPARMv8,
                                  FeatureFullFP16, FeatureJS, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> A715 = [HasV9_0aOps, FeatureNEON, FeatureMTE,
+                                 FeatureCCIDX,
                                  FeatureFP16FML, FeatureSVE, FeatureTRBE,
                                  FeatureSVE2BitPerm, FeatureBF16, FeatureETE,
                                  FeaturePerfMon, FeatureMatMulInt8, FeatureSPE,
@@ -749,6 +754,7 @@ def ProcessorFeatures {
                                  FeatureJS, FeatureLSE, FeatureRAS,
                                  FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> A720 = [HasV9_2aOps, FeatureMTE, FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureTRBE, FeatureSVE2BitPerm, FeatureETE,
                                  FeaturePerfMon, FeatureSPE, FeatureSPE_EEF,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
@@ -757,6 +763,7 @@ def ProcessorFeatures {
                                  FeatureJS, FeatureLSE, FeatureNEON, FeatureRAS,
                                  FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> A720AE = [HasV9_2aOps, FeatureMTE, FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureTRBE, FeatureSVE2BitPerm, FeatureETE,
                                  FeaturePerfMon, FeatureSPE, FeatureSPE_EEF,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
@@ -765,6 +772,7 @@ def ProcessorFeatures {
                                  FeatureJS, FeatureLSE, FeatureNEON, FeatureRAS,
                                  FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> A725 = [HasV9_2aOps, FeatureMTE, FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureETE, FeaturePerfMon, FeatureSPE,
                                  FeatureSVE2BitPerm, FeatureSPE_EEF, FeatureTRBE,
                                  FeatureFlagM, FeaturePredRes, FeatureSB, FeatureSSBS,
@@ -800,6 +808,7 @@ def ProcessorFeatures {
                                  FeatureMatMulInt8, FeatureBF16, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeaturePAuth, FeatureSSBS, FeatureSB, FeatureSVE, FeatureSVE2, FeatureFlagM,
                                  FeatureComplxNum, FeatureCRC, FeatureDotProd, FeatureFPARMv8, FeatureFullFP16,
                                  FeatureJS, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM];
@@ -808,6 +817,7 @@ def ProcessorFeatures {
                                  FeatureSPE, FeatureBF16, FeatureMatMulInt8,
                                  FeatureMTE, FeatureSVE2BitPerm, FeatureFullFP16,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureSB, FeaturePAuth, FeaturePredRes, FeatureFlagM, FeatureSSBS,
                                  FeatureSVE2, FeatureComplxNum, FeatureCRC, FeatureFPARMv8, FeatureJS,
                                  FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM, FeatureDotProd];
@@ -815,11 +825,13 @@ def ProcessorFeatures {
                                  FeaturePerfMon, FeatureETE, FeatureTRBE,
                                  FeatureSPE, FeatureMTE, FeatureSVE2BitPerm,
                                  FeatureFP16FML, FeatureSPE_EEF,
+                                 FeatureCCIDX,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
                                  FeatureSVE, FeatureSVE2, FeatureComplxNum, FeatureCRC, FeatureDotProd,
                                  FeatureFPARMv8, FeatureFullFP16, FeatureMatMulInt8, FeatureJS, FeatureLSE,
                                  FeatureNEON, FeatureRAS, FeatureRCPC, FeatureRDM, FeatureBF16];
   list<SubtargetFeature> X925 = [HasV9_2aOps, FeatureMTE, FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureETE, FeaturePerfMon, FeatureSPE,
                                  FeatureSVE2BitPerm, FeatureSPE_EEF, FeatureTRBE,
                                  FeatureFlagM, FeaturePredRes, FeatureSB, FeatureSSBS,
@@ -835,7 +847,7 @@ def ProcessorFeatures {
                                      FeatureFullFP16, FeatureCRC, FeatureLSE, FeatureRAS, FeatureRDM,
                                      FeatureFPARMv8];
   list<SubtargetFeature> AppleA7  = [HasV8_0aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
-                                     FeatureNEON,FeaturePerfMon, FeatureAppleA7SysReg];
+                                     FeatureNEON,FeaturePerfMon];
   list<SubtargetFeature> AppleA10 = [HasV8_0aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureCRC,
                                      FeatureRDM, FeaturePAN, FeatureLOR, FeatureVH];
@@ -866,7 +878,7 @@ def ProcessorFeatures {
                                      FeatureComplxNum, FeatureCRC, FeatureJS,
                                      FeatureLSE, FeaturePAuth, FeatureFPAC,
                                      FeatureRAS, FeatureRCPC, FeatureRDM,
-                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8];
+                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8, FeatureSSBS];
   list<SubtargetFeature> AppleA16 = [HasV8_6aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureSHA3,
                                      FeatureFullFP16, FeatureFP16FML,
@@ -874,7 +886,7 @@ def ProcessorFeatures {
                                      FeatureComplxNum, FeatureCRC, FeatureJS,
                                      FeatureLSE, FeaturePAuth, FeatureFPAC,
                                      FeatureRAS, FeatureRCPC, FeatureRDM,
-                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8];
+                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8, FeatureSSBS];
   list<SubtargetFeature> AppleA17 = [HasV8_6aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureSHA3,
                                      FeatureFullFP16, FeatureFP16FML,
@@ -882,7 +894,7 @@ def ProcessorFeatures {
                                      FeatureComplxNum, FeatureCRC, FeatureJS,
                                      FeatureLSE, FeaturePAuth, FeatureFPAC,
                                      FeatureRAS, FeatureRCPC, FeatureRDM,
-                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8];
+                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8, FeatureSSBS];
   list<SubtargetFeature> AppleM4 = [HasV9_2aOps, FeatureSHA2, FeatureFPARMv8,
                                     FeatureNEON, FeaturePerfMon, FeatureSHA3,
                                     FeatureFullFP16, FeatureFP16FML,
@@ -913,6 +925,7 @@ def ProcessorFeatures {
                                        FeatureMatMulInt8, FeatureMTE, FeatureSVE2,
                                        FeatureSVE2BitPerm, FeatureTRBE,
                                        FeaturePerfMon,
+                                       FeatureCCIDX,
                                        FeatureDotProd, FeatureFullFP16, FeatureSB, FeatureSSBS, FeatureSVE,
                                        FeatureComplxNum, FeatureCRC, FeatureFPARMv8, FeatureJS, FeatureLSE,
                                        FeatureNEON, FeaturePAuth, FeatureRAS, FeatureRCPC, FeatureRDM];
@@ -920,6 +933,7 @@ def ProcessorFeatures {
                                       FeatureFullFP16, FeatureMTE, FeaturePerfMon,
                                       FeatureRandGen, FeatureSPE, FeatureSPE_EEF,
                                       FeatureSVE2BitPerm,
+                                      FeatureCCIDX,
                                       FeatureSSBS, FeatureSB, FeaturePredRes, FeaturePAuth, FeatureFlagM,
                                       FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum,
                                       FeatureCRC, FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
@@ -930,6 +944,7 @@ def ProcessorFeatures {
                                            FeatureFullFP16, FeatureMatMulInt8, FeatureNEON,
                                            FeaturePerfMon, FeatureRandGen, FeatureSPE,
                                            FeatureSSBS, FeatureSVE,
+                                           FeatureCCIDX,
                                            FeatureSHA3, FeatureSM4, FeatureDotProd, FeatureComplxNum,
                                            FeatureCRC, FeatureJS, FeatureLSE, FeaturePAuth, FeatureRAS,
                                            FeatureRCPC, FeatureRDM];
@@ -938,6 +953,7 @@ def ProcessorFeatures {
                                        FeatureFullFP16, FeatureMatMulInt8, FeatureNEON,
                                        FeaturePerfMon, FeatureRandGen, FeatureSPE,
                                        FeatureSSBS, FeatureSVE,
+                                       FeatureCCIDX,
                                        FeatureSHA3, FeatureSM4, FeatureDotProd, FeatureComplxNum,
                                        FeatureCRC, FeatureJS, FeatureLSE, FeaturePAuth, FeatureRAS,
                                        FeatureRCPC, FeatureRDM];
@@ -945,12 +961,14 @@ def ProcessorFeatures {
                                        FeaturePerfMon, FeatureETE, FeatureMatMulInt8,
                                        FeatureNEON, FeatureSVE2BitPerm, FeatureFP16FML,
                                        FeatureMTE, FeatureRandGen,
+                                       FeatureCCIDX,
                                        FeatureSVE, FeatureSVE2, FeatureSSBS, FeatureFullFP16, FeatureDotProd,
                                        FeatureComplxNum, FeatureCRC, FeatureFPARMv8, FeatureJS, FeatureLSE,
                                        FeaturePAuth, FeatureRAS, FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> NeoverseV3 = [HasV9_2aOps, FeatureETE, FeatureFP16FML,
                                       FeatureFullFP16, FeatureLS64, FeatureMTE,
                                       FeaturePerfMon, FeatureRandGen, FeatureSPE,
+                                      FeatureCCIDX,
                                       FeatureSPE_EEF, FeatureSVE2BitPerm, FeatureBRBE,
                                       FeatureSSBS, FeatureSB, FeaturePredRes, FeaturePAuth, FeatureFlagM,
                                       FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
@@ -961,12 +979,14 @@ def ProcessorFeatures {
                                       FeaturePerfMon, FeatureRandGen, FeatureSPE,
                                       FeatureSPE_EEF, FeatureSVE2BitPerm, FeatureBRBE,
                                       FeatureSSBS, FeatureSB, FeaturePredRes, FeaturePAuth, FeatureFlagM,
+                                      FeatureCCIDX,
                                       FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                       FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8, FeatureJS,
                                       FeatureLSE, FeatureNEON, FeatureRAS, FeatureRCPC, FeatureRDM,
                                       FeatureRME];
   list<SubtargetFeature> Saphira    = [HasV8_4aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                        FeatureNEON, FeatureSPE, FeaturePerfMon, FeatureCRC,
+                                       FeatureCCIDX,
                                        FeatureLSE, FeatureRDM, FeatureRAS, FeatureRCPC];
   list<SubtargetFeature> ThunderX   = [HasV8_0aOps, FeatureCRC, FeatureSHA2, FeatureAES,
                                        FeatureFPARMv8, FeaturePerfMon, FeatureNEON];
@@ -975,6 +995,7 @@ def ProcessorFeatures {
                                           FeatureRDM];
   list<SubtargetFeature> ThunderX3T110 = [HasV8_3aOps, FeatureCRC, FeatureSHA2, FeatureAES,
                                           FeatureFPARMv8, FeatureNEON, FeatureLSE,
+                                          FeatureCCIDX,
                                           FeaturePAuth, FeaturePerfMon, FeatureComplxNum,
                                           FeatureJS, FeatureRAS, FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> TSV110 = [HasV8_2aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
@@ -987,6 +1008,7 @@ def ProcessorFeatures {
                                     FeatureSHA2, FeatureSHA3, FeatureAES,
                                     FeatureFullFP16, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                     FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8, FeatureJS,
+                                    FeatureCCIDX,
                                     FeatureLSE, FeaturePAuth, FeatureRAS, FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> Ampere1A = [HasV8_6aOps, FeatureNEON, FeaturePerfMon,
                                      FeatureMTE, FeatureSSBS, FeatureRandGen,
@@ -995,6 +1017,7 @@ def ProcessorFeatures {
                                      FeatureFullFP16, FeatureBF16, FeatureComplxNum,
                                      FeatureCRC, FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
                                      FeatureJS, FeatureLSE, FeaturePAuth, FeatureRAS, FeatureRCPC,
+                                     FeatureCCIDX,
                                      FeatureRDM];
   list<SubtargetFeature> Ampere1B = [HasV8_7aOps, FeatureNEON, FeaturePerfMon,
                                      FeatureMTE, FeatureSSBS, FeatureRandGen,
@@ -1003,6 +1026,7 @@ def ProcessorFeatures {
                                      FeatureWFxT, FeatureFullFP16, FeatureBF16, FeatureComplxNum,
                                      FeatureCRC, FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
                                      FeatureJS, FeatureLSE, FeaturePAuth, FeatureRAS, FeatureRCPC,
+                                     FeatureCCIDX,
                                      FeatureRDM];
 
   list<SubtargetFeature> Oryon = [HasV8_6aOps, FeatureNEON, FeaturePerfMon,
@@ -1011,6 +1035,7 @@ def ProcessorFeatures {
                                      FeatureSHA3, FeatureAES,
                                      FeatureSPE, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                      FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
+                                     FeatureSSBS, FeatureCCIDX,
                                      FeatureJS, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM];
 
   // ETE and TRBE are future architecture extensions. We temporarily enable them
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 435cc18cdea6249..2deffbd282569c1 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -479,6 +479,7 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
   }
 
   markSuperRegs(Reserved, AArch64::FPCR);
+  markSuperRegs(Reserved, AArch64::FPMR);
   markSuperRegs(Reserved, AArch64::FPSR);
 
   if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index 4dc33e6168cbdaf..f754c32e1176d1d 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -147,6 +147,9 @@ def VG : AArch64Reg<0, "vg">, DwarfRegNum<[46]>;
 // Floating-point control register
 def FPCR : AArch64Reg<0, "fpcr">;
 
+// Floating-point Mode Register
+def FPMR : AArch64Reg<0, "fpmr">;
+
 // Floating-point status register.
 def FPSR : AArch64Reg<0, "fpsr">;
 
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index d9a70b5ef02fcb1..3f4651ea9c2b68f 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4244,19 +4244,18 @@ def FMLALLTT_ZZZ : sve2_fp8_mla<0b011, ZPR32, "fmlalltt">;
 
 let Predicates = [HasSSVE_FP8DOT2] in {
 // FP8 Widening Dot-Product - Indexed Group
-defm FDOT_ZZZI_BtoH : sve2_fp8_dot_indexed<"fdot">;
+defm FDOT_ZZZI_BtoH : sve2_fp8_dot_indexed_h<"fdot">;
 // FP8 Widening Dot-Product - Group
 // TODO: Replace nxv16i8 by nxv16f8
-defm FDOT_ZZZ_BtoH : sve_float_dot<0b0, 0b1, ZPR16, ZPR8, "fdot", nxv16i8, null_frag>;
+defm FDOT_ZZZ_BtoH : sve_fp8_dot<0b0, ZPR16, "fdot">;
 }
 
 // TODO: Replace nxv16i8 by nxv16f8
 let Predicates = [HasSSVE_FP8DOT4] in {
 // FP8 Widening Dot-Product - Indexed Group
-defm FDOT_ZZZI_BtoS : sve_float_dot_indexed<0b1, 0b01, ZPR8, ZPR3b8, "fdot",
-                                            nxv16i8, null_frag>;
+defm FDOT_ZZZI_BtoS : sve2_fp8_dot_indexed_s<"fdot">;
 // FP8 Widening Dot-Product - Group
-defm FDOT_ZZZ_BtoS : sve_float_dot<0b1, 0b1, ZPR32, ZPR8, "fdot", nxv16i8, null_frag>;
+defm FDOT_ZZZ_BtoS : sve_fp8_dot<0b1, ZPR32, "fdot">;
 }
 
 let Predicates = [HasSVE2orSME2, HasLUT] in {
diff --git a/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
index cd6e6bffda10bca..7476ab852a923bd 100644
--- a/llvm/lib/Target/AArch64/AArch64SystemOperands.td
+++ b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
@@ -1747,11 +1747,6 @@ foreach n = 0-31 in {
 let Requires = [{ {AArch64::FeatureSPE_EEF} }] in
 def : RWSysReg<"PMSNEVFR_EL1",      0b11, 0b000, 0b1001, 0b1001, 0b001>;
 
-// Cyclone specific system registers
-//                                 Op0    Op1     CRn     CRm    Op2
-let Requires = [{ {AArch64::FeatureAppleA7SysReg} }] in
-def : RWSysReg<"CPM_IOACC_CTL_EL3", 0b11, 0b111, 0b1111, 0b0010, 0b000>;
-
 // Scalable Matrix Extension (SME)
 //                                 Op0   Op1    CRn     CRm     Op2
 let Requires = [{ {AArch64::FeatureSME} }] in {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index a416565392eabec..a782c9c43512379 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2165,6 +2165,24 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
   case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
   case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
     return instCombineSVENoActiveUnaryZero(IC, II);
+  case Intrinsic::aarch64_sve_prf:
+  case Intrinsic::aarch64_sve_prfb_gather_index:
+  case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
+  case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
+  case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
+  case Intrinsic::aarch64_sve_prfd_gather_index:
+  case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
+  case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
+  case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
+  case Intrinsic::aarch64_sve_prfh_gather_index:
+  case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
+  case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
+  case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
+  case Intrinsic::aarch64_sve_prfw_gather_index:
+  case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
+  case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
+  case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
+    return instCombineSVENoActiveUnaryErase(IC, II, 0);
   case Intrinsic::aarch64_neon_fmaxnm:
   case Intrinsic::aarch64_neon_fminnm:
     return instCombineMaxMinNM(IC, II);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index f71fe323a6d3581..28d9f4f50f3883a 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -566,6 +566,11 @@ bool AArch64PostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
   CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
                      /*LegalizerInfo*/ nullptr, EnableOpt, F.hasOptSize(),
                      F.hasMinSize());
+  // Disable fixed-point iteration to reduce compile-time
+  CInfo.MaxIterations = 1;
+  CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
+  // Legalizer performs DCE, so a full DCE pass is unnecessary.
+  CInfo.EnableFullDCE = false;
   AArch64PostLegalizerCombinerImpl Impl(MF, CInfo, TPC, *KB, CSEInfo,
                                         RuleConfig, ST, MDT, LI);
   bool Changed = Impl.combineMachineInstrs();
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index afea8bdec1973a1..90ac4bdff4e0e48 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -1296,6 +1296,11 @@ bool AArch64PostLegalizerLowering::runOnMachineFunction(MachineFunction &MF) {
   CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
                      /*LegalizerInfo*/ nullptr, /*OptEnabled=*/true,
                      F.hasOptSize(), F.hasMinSize());
+  // Disable fixed-point iteration to reduce compile-time
+  CInfo.MaxIterations = 1;
+  CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
+  // PostLegalizerCombiner performs DCE, so a full DCE pass is unnecessary.
+  CInfo.EnableFullDCE = false;
   AArch64PostLegalizerLoweringImpl Impl(MF, CInfo, TPC, /*CSEInfo*/ nullptr,
                                         RuleConfig, ST);
   return Impl.combineMachineInstrs();
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index 8a50cb26b2c2fee..6e689d743804ac9 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -861,6 +861,12 @@ bool AArch64PreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
   CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
                      /*LegalizerInfo*/ nullptr, EnableOpt, F.hasOptSize(),
                      F.hasMinSize());
+  // Disable fixed-point iteration to reduce compile-time
+  CInfo.MaxIterations = 1;
+  CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
+  // This is the first Combiner, so the input IR might contain dead
+  // instructions.
+  CInfo.EnableFullDCE = true;
   AArch64PreLegalizerCombinerImpl Impl(MF, CInfo, &TPC, *KB, CSEInfo,
                                        RuleConfig, ST, MDT, LI);
   return Impl.combineMachineInstrs();
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index ed670bce594ec60..c69c87c685303c7 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -272,8 +272,6 @@ class AArch64ELFStreamer : public MCELFStreamer {
   void emitMappingSymbol(StringRef Name) {
     auto *Symbol = cast<MCSymbolELF>(getContext().createLocalSymbol(Name));
     emitLabel(Symbol);
-    Symbol->setType(ELF::STT_NOTYPE);
-    Symbol->setBinding(ELF::STB_LOCAL);
   }
 
   DenseMap<const MCSection *, ElfMappingSymbol> LastMappingSymbols;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
index 33f0ca237b26dd1..ed0a972cc50abdc 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/AArch64FixupKinds.h"
+#include "MCTargetDesc/AArch64MCExpr.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -394,6 +395,46 @@ void AArch64MachObjectWriter::recordRelocation(
     Value = 0;
   }
 
+  if (Target.getRefKind() == AArch64MCExpr::VK_AUTH ||
+      Target.getRefKind() == AArch64MCExpr::VK_AUTHADDR) {
+    auto *Expr = cast<AArch64AuthMCExpr>(Fixup.getValue());
+
+    assert(Type == MachO::ARM64_RELOC_UNSIGNED);
+
+    if (IsPCRel) {
+      Asm.getContext().reportError(Fixup.getLoc(),
+                                   "invalid PC relative auth relocation");
+      return;
+    }
+
+    if (Log2Size != 3) {
+      Asm.getContext().reportError(
+          Fixup.getLoc(), "invalid auth relocation size, must be 8 bytes");
+      return;
+    }
+
+    if (Target.getSymB()) {
+      Asm.getContext().reportError(
+          Fixup.getLoc(),
+          "invalid auth relocation, can't reference two symbols");
+      return;
+    }
+
+    uint16_t Discriminator = Expr->getDiscriminator();
+    AArch64PACKey::ID Key = Expr->getKey();
+
+    if (!isInt<32>(Value)) {
+      Asm.getContext().reportError(Fixup.getLoc(),
+                                   "addend too big for relocation");
+      return;
+    }
+
+    Type = MachO::ARM64_RELOC_AUTHENTICATED_POINTER;
+    Value = (uint32_t(Value)) | (uint64_t(Discriminator) << 32) |
+            (uint64_t(Expr->hasAddressDiversity()) << 48) |
+            (uint64_t(Key) << 49) | (1ULL << 63);
+  }
+
   // If there's any addend left to handle, encode it in the instruction.
   FixedValue = Value;
 
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 77cf5cb56728b91..a83c132496efadf 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -2240,7 +2240,9 @@ multiclass sme2_cvt_vg2_single<string mnemonic, bits<5> op, ValueType out_vt,
 
 // SME2 multi-vec FP8 down convert two registers
 multiclass sme2_fp8_cvt_vg2_single<string mnemonic, bit op> {
-  def NAME :  sme2_cvt_vg2_single<mnemonic, {op, 0b1000}, ZPR8, ZZ_h_mul_r>;
+  def NAME :  sme2_cvt_vg2_single<mnemonic, {op, 0b1000}, ZPR8, ZZ_h_mul_r>{
+    let Uses = [FPMR, FPCR];
+  }
 }
 
 class sme2_cvt_unpk_vector_vg2<bits<2>sz, bits<3> op, bit u, RegisterOperand first_ty,
@@ -2273,7 +2275,9 @@ multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> {
 
 // SME2 multi-vec FP8 up convert two registers
 multiclass sme2p1_fp8_cvt_vector_vg2_single<string mnemonic, bits<2> opc, bit L> {
-  def _NAME : sme2_cvt_unpk_vector_vg2<opc, 0b110, L, ZZ_h_mul_r, ZPR8, mnemonic>;
+  def _NAME : sme2_cvt_unpk_vector_vg2<opc, 0b110, L, ZZ_h_mul_r, ZPR8, mnemonic>{
+    let Uses = [FPMR, FPCR];
+  }
 }
 
 
@@ -2536,6 +2540,7 @@ class sme2_fp8_multi_vec_array_vg4_index<string mnemonic, bit T>
   let Inst{10} = i{1};
   let Inst{3}  = i{0};
   let AsmString = !strconcat(mnemonic, "{\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i}");
+  let Uses = [FPMR, FPCR];
 }
 
 // SME2 multi-vec ternary indexed two registers 64-bit
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 482d3a0664d72fb..107bc79f70dbcb5 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -8708,6 +8708,12 @@ multiclass sve_float_dot<bit bf, bit o2, ZPRRegOp dst_ty, ZPRRegOp src_ty,
   def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, InVT, InVT, !cast<Instruction>(NAME)>;
 }
 
+multiclass sve_fp8_dot<bit bf, ZPRRegOp dst_ty, string asm> {
+  def NAME : sve_float_dot<bf, 0b1, dst_ty, ZPR8, asm>{
+    let Uses = [FPMR, FPCR];
+  }
+}
+
 class sve_float_dot_indexed<bit bf, ZPRRegOp dst_ty, ZPRRegOp src1_ty,
                             ZPRRegOp src2_ty, Operand iop_ty, string asm>
 : I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, src1_ty:$Zn, src2_ty:$Zm, iop_ty:$iop),
@@ -10212,6 +10218,7 @@ class sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic,
   let Inst{11-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+  let Uses = [FPMR, FPCR];
 }
 
 multiclass sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic> {
@@ -10231,6 +10238,7 @@ class sve2_fp8_down_cvt_single<bits<2> opc, string mnemonic,
   let Inst{9-6} = Zn;
   let Inst{5} = 0b0;
   let Inst{4-0} = Zd;
+  let Uses = [FPMR, FPCR];
 }
 
 multiclass sve2_fp8_down_cvt_single<bits<2> opc, string mnemonic, RegisterOperand src> {
@@ -10259,6 +10267,7 @@ class sve2_fp8_mla_long_by_indexed_elem<bit T, string mnemonic>
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize         = ZPR16.ElementSize;
+  let Uses = [FPMR, FPCR];
 }
 
 // FP8 Widening Multiply-Add (Long)/(Long Long) Group
@@ -10282,6 +10291,7 @@ class sve2_fp8_mla<bits<3>opc, ZPRRegOp dst_ty, string mnemonic>
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize         = dst_ty.ElementSize;
+  let Uses = [FPMR, FPCR];
 }
 
 // FP8 Widening Multiply-Add Long Long - Indexed Group
@@ -10306,15 +10316,48 @@ class sve2_fp8_mla_long_long_by_indexed_elem<bits<2> TT, string mnemonic>
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize         = ZPR32.ElementSize;
+  let Uses = [FPMR, FPCR];
+}
+
+class sve_fp8_dot_indexed<bits<4> opc, ZPRRegOp dst_ty, Operand iop_ty, string mnemonic>
+: I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, iop_ty:$iop),
+    mnemonic, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {
+  bits<5> Zda;
+  bits<5> Zn;
+  bits<3> Zm;
+  let Inst{31-23} = 0b011001000;
+  let Inst{22}    = opc{3};
+  let Inst{21}    = 0b1;
+  let Inst{20-19} = opc{2-1};
+  let Inst{18-16} = Zm;
+  let Inst{15-12} = 0b0100;
+  let Inst{11}    = opc{0};
+  let Inst{10}    = 0b1;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zda;
+
+  let Uses = [FPMR, FPCR];
+  let Constraints = "$Zda = $_Zda";
+  let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 // FP8 Widening Dot-Product - Indexed Group
-multiclass sve2_fp8_dot_indexed<string mnemonic>{
-  def NAME : sve_float_dot_indexed<0b0, ZPR16, ZPR8, ZPR3b8, VectorIndexH, mnemonic> {
+multiclass sve2_fp8_dot_indexed_h<string asm>{
+  def NAME : sve_fp8_dot_indexed<{0, ?, ?, ?}, ZPR16, VectorIndexH, asm> {
     bits<3> iop;
+
     let Inst{20-19} = iop{2-1};
-    let Inst{11} = iop{0};
-    let Inst{10} = 0b1;
+    let Inst{11}    = iop{0};
+  }
+}
+
+multiclass sve2_fp8_dot_indexed_s<string asm>{
+  def NAME : sve_fp8_dot_indexed<{1, ?, ?, 0}, ZPR32, VectorIndexS32b, asm> {
+    bits<2> iop;
+
+    let Inst{20-19} = iop{1-0};
   }
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index e64e28e01d3d185..b90d245b7bd3944 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -395,12 +395,10 @@ void AMDGPUAsmPrinter::emitCommonFunctionComments(
 SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
   SmallString<128> Str;
   raw_svector_ostream OSS(Str);
-  int64_t IVal;
-  if (Value->evaluateAsAbsolute(IVal)) {
-    OSS << static_cast<uint64_t>(IVal);
-  } else {
-    Value->print(OSS, MAI);
-  }
+  auto &Streamer = getTargetStreamer()->getStreamer();
+  auto &Context = Streamer.getContext();
+  const MCExpr *New = foldAMDGPUMCExpr(Value, Context);
+  printAMDGPUMCExpr(New, OSS, MAI);
   return Str;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp
deleted file mode 100644
index 36f44a20d955320..000000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-//===- lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPUCodeGenPassBuilder.h"
-#include "AMDGPU.h"
-#include "AMDGPUISelDAGToDAG.h"
-#include "AMDGPUPerfHintAnalysis.h"
-#include "AMDGPUTargetMachine.h"
-#include "AMDGPUUnifyDivergentExitNodes.h"
-#include "SIFixSGPRCopies.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Transforms/Scalar/FlattenCFG.h"
-#include "llvm/Transforms/Scalar/Sink.h"
-#include "llvm/Transforms/Scalar/StructurizeCFG.h"
-#include "llvm/Transforms/Utils/FixIrreducible.h"
-#include "llvm/Transforms/Utils/LCSSA.h"
-#include "llvm/Transforms/Utils/UnifyLoopExits.h"
-
-using namespace llvm;
-
-AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
-    GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
-    PassInstrumentationCallbacks *PIC)
-    : CodeGenPassBuilder(TM, Opts, PIC) {
-  Opt.RequiresCodeGenSCCOrder = true;
-  // Exceptions and StackMaps are not supported, so these passes will never do
-  // anything.
-  // Garbage collection is not supported.
-  disablePass<StackMapLivenessPass, FuncletLayoutPass,
-              ShadowStackGCLoweringPass>();
-}
-
-void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
-  const bool LateCFGStructurize = AMDGPUTargetMachine::EnableLateStructurizeCFG;
-  const bool DisableStructurizer = AMDGPUTargetMachine::DisableStructurizer;
-  const bool EnableStructurizerWorkarounds =
-      AMDGPUTargetMachine::EnableStructurizerWorkarounds;
-
-  if (TM.getOptLevel() > CodeGenOptLevel::None)
-    addPass(FlattenCFGPass());
-
-  if (TM.getOptLevel() > CodeGenOptLevel::None)
-    addPass(SinkingPass());
-
-  addPass(AMDGPULateCodeGenPreparePass(TM));
-
-  // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
-  // regions formed by them.
-
-  addPass(AMDGPUUnifyDivergentExitNodesPass());
-
-  if (!LateCFGStructurize && !DisableStructurizer) {
-    if (EnableStructurizerWorkarounds) {
-      addPass(FixIrreduciblePass());
-      addPass(UnifyLoopExitsPass());
-    }
-
-    addPass(StructurizeCFGPass(/*SkipUniformRegions=*/false));
-  }
-
-  addPass(AMDGPUAnnotateUniformValuesPass());
-
-  if (!LateCFGStructurize && !DisableStructurizer) {
-    addPass(SIAnnotateControlFlowPass(TM));
-
-    // TODO: Move this right after structurizeCFG to avoid extra divergence
-    // analysis. This depends on stopping SIAnnotateControlFlow from making
-    // control flow modifications.
-    addPass(AMDGPURewriteUndefForPHIPass());
-  }
-
-  addPass(LCSSAPass());
-
-  if (TM.getOptLevel() > CodeGenOptLevel::Less)
-    addPass(AMDGPUPerfHintAnalysisPass(TM));
-
-  // FIXME: Why isn't this queried as required from AMDGPUISelDAGToDAG, and why
-  // isn't this in addInstSelector?
-  addPass(RequireAnalysisPass<UniformityInfoAnalysis, Function>());
-}
-
-void AMDGPUCodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,
-                                             CreateMCStreamer) const {
-  // TODO: Add AsmPrinter.
-}
-
-Error AMDGPUCodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const {
-  addPass(AMDGPUISelDAGToDAGPass(TM));
-  addPass(SIFixSGPRCopiesPass());
-  addPass(SILowerI1CopiesPass());
-  return Error::success();
-}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h
deleted file mode 100644
index e656e166b3eb2e2..000000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h
+++ /dev/null
@@ -1,33 +0,0 @@
-//===- lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h -----------*- C++ -*---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCODEGENPASSBUILDER_H
-#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCODEGENPASSBUILDER_H
-
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/Passes/CodeGenPassBuilder.h"
-
-namespace llvm {
-
-class GCNTargetMachine;
-
-class AMDGPUCodeGenPassBuilder
-    : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {
-public:
-  AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
-                           const CGPassBuilderOption &Opts,
-                           PassInstrumentationCallbacks *PIC);
-
-  void addPreISel(AddIRPass &addPass) const;
-  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
-  Error addInstSelector(AddMachinePass &) const;
-};
-
-} // namespace llvm
-
-#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUCODEGENPASSBUILDER_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 825c1de6c91f12b..8bee84b8a87f276 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -297,8 +297,7 @@ def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_UBYTE, SIsbuffer_load_ubyte>;
 def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_SSHORT, SIsbuffer_load_short>;
 def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_USHORT, SIsbuffer_load_ushort>;
 
-def : GINodeEquiv<G_FPTRUNC_ROUND_UPWARD, SIfptrunc_round_upward>;
-def : GINodeEquiv<G_FPTRUNC_ROUND_DOWNWARD, SIfptrunc_round_downward>;
+def : GINodeEquiv<G_FPTRUNC_ROUND, SIfptrunc_round>;
 
 class GISelSop2Pat <
   SDPatternOperator node,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 8c8950957dd8197..e57c8f8b7b4835c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5500,8 +5500,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(CONST_DATA_PTR)
   NODE_NAME_CASE(PC_ADD_REL_OFFSET)
   NODE_NAME_CASE(LDS)
-  NODE_NAME_CASE(FPTRUNC_ROUND_UPWARD)
-  NODE_NAME_CASE(FPTRUNC_ROUND_DOWNWARD)
+  NODE_NAME_CASE(FPTRUNC_ROUND)
   NODE_NAME_CASE(DUMMY_CHAIN)
   case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
   NODE_NAME_CASE(LOAD_D16_HI)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 1a5244f7ec809b8..59f640ea99de3ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -553,8 +553,7 @@ enum NodeType : unsigned {
   CONST_DATA_PTR,
   PC_ADD_REL_OFFSET,
   LDS,
-  FPTRUNC_ROUND_UPWARD,
-  FPTRUNC_ROUND_DOWNWARD,
+  FPTRUNC_ROUND,
 
   DUMMY_CHAIN,
   FIRST_MEM_OPCODE_NUMBER = ISD::FIRST_TARGET_MEMORY_OPCODE,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c6c4b8f93064711..4fd917f5ea7fa80 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2368,25 +2368,30 @@ bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
   if (DestAS == AMDGPUAS::FLAT_ADDRESS &&
       (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
        SrcAS == AMDGPUAS::PRIVATE_ADDRESS)) {
-    Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
-    if (!ApertureReg.isValid())
-      return false;
+    auto castLocalOrPrivateToFlat = [&](const DstOp &Dst) -> Register {
+      Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
+      if (!ApertureReg.isValid())
+        return false;
 
-    // Coerce the type of the low half of the result so we can use merge_values.
-    Register SrcAsInt = B.buildPtrToInt(S32, Src).getReg(0);
+      // Coerce the type of the low half of the result so we can use
+      // merge_values.
+      Register SrcAsInt = B.buildPtrToInt(S32, Src).getReg(0);
 
-    // TODO: Should we allow mismatched types but matching sizes in merges to
-    // avoid the ptrtoint?
-    auto BuildPtr = B.buildMergeLikeInstr(DstTy, {SrcAsInt, ApertureReg});
+      // TODO: Should we allow mismatched types but matching sizes in merges to
+      // avoid the ptrtoint?
+      return B.buildMergeLikeInstr(Dst, {SrcAsInt, ApertureReg}).getReg(0);
+    };
 
     // For llvm.amdgcn.addrspacecast.nonnull we can always assume non-null, for
     // G_ADDRSPACE_CAST we need to guess.
     if (isa<GIntrinsic>(MI) || isKnownNonNull(Src, MRI, TM, SrcAS)) {
-      B.buildCopy(Dst, BuildPtr);
+      castLocalOrPrivateToFlat(Dst);
       MI.eraseFromParent();
       return true;
     }
 
+    Register BuildPtr = castLocalOrPrivateToFlat(DstTy);
+
     auto SegmentNull = B.buildConstant(SrcTy, TM.getNullPointerValue(SrcAS));
     auto FlatNull = B.buildConstant(DstTy, TM.getNullPointerValue(DestAS));
 
@@ -7090,22 +7095,30 @@ bool AMDGPULegalizerInfo::legalizeBVHIntrinsic(MachineInstr &MI,
 
 bool AMDGPULegalizerInfo::legalizeFPTruncRound(MachineInstr &MI,
                                                MachineIRBuilder &B) const {
-  unsigned Opc;
-  int RoundMode = MI.getOperand(2).getImm();
+  MachineRegisterInfo &MRI = *B.getMRI();
+  Register Src = MI.getOperand(1).getReg();
+  if (MRI.getType(Src) != LLT::scalar(32))
+    return false;
 
-  if (RoundMode == (int)RoundingMode::TowardPositive)
-    Opc = AMDGPU::G_FPTRUNC_ROUND_UPWARD;
-  else if (RoundMode == (int)RoundingMode::TowardNegative)
-    Opc = AMDGPU::G_FPTRUNC_ROUND_DOWNWARD;
-  else
+  // Only support towardzero, tonearest, upward and downward.
+  int RoundMode = MI.getOperand(2).getImm();
+  if (RoundMode != (int)RoundingMode::TowardZero &&
+      RoundMode != (int)RoundingMode::NearestTiesToEven &&
+      RoundMode != (int)RoundingMode::TowardPositive &&
+      RoundMode != (int)RoundingMode::TowardNegative)
     return false;
 
-  B.buildInstr(Opc)
+  // "round.towardzero" -> TowardZero 0        -> FP_ROUND_ROUND_TO_ZERO 3
+  // "round.tonearest"  -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
+  // "round.upward"     -> TowardPositive 2    -> FP_ROUND_ROUND_TO_INF 1
+  // "round.downward    -> TowardNegative 3    -> FP_ROUND_ROUND_TO_NEGINF 2
+  unsigned HW_Mode = (RoundMode + 3) % 4;
+  B.buildInstr(AMDGPU::G_FPTRUNC_ROUND)
       .addDef(MI.getOperand(0).getReg())
-      .addUse(MI.getOperand(1).getReg());
+      .addUse(Src)
+      .addImm(HW_Mode);
 
   MI.eraseFromParent();
-
   return true;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 17067ddd93ff085..5add368c05646a1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -5259,8 +5259,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1);
     break;
   }
-  case AMDGPU::G_FPTRUNC_ROUND_UPWARD:
-  case AMDGPU::G_FPTRUNC_ROUND_DOWNWARD:
+  case AMDGPU::G_FPTRUNC_ROUND:
     return getDefaultMappingVOP(MI);
   case AMDGPU::G_PREFETCH:
     OpdsMapping[0] = getSGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
index bd0f0e048809bcd..df084cf41c47836 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
@@ -568,12 +568,13 @@ doPartitioning(SplitModuleLogger &SML, Module &M, unsigned NumParts,
   }
 
   if (SML) {
+    CostType ModuleCostOr1 = ModuleCost ? ModuleCost : 1;
     for (const auto &[Idx, Part] : enumerate(Partitions)) {
       CostType Cost = 0;
       for (auto *Fn : Part)
         Cost += FnCosts.at(Fn);
       SML << "P" << Idx << " has a total cost of " << Cost << " ("
-          << format("%0.2f", (float(Cost) / ModuleCost) * 100)
+          << format("%0.2f", (float(Cost) / ModuleCostOr1) * 100)
           << "% of source module)\n";
     }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index b8aa93285ad849e..04b96a32a8970ae 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -7,15 +7,16 @@
 //===----------------------------------------------------------------------===//
 //
 /// \file
-/// The AMDGPU target machine contains all of the hardware specific
-/// information  needed to emit code for SI+ GPUs.
+/// This file contains both AMDGPU target machine and the CodeGen pass builder.
+/// The AMDGPU target machine contains all of the hardware specific information
+/// needed to emit code for SI+ GPUs in the legacy pass manager pipeline. The
+/// CodeGen pass builder handles the pass pipeline for new pass manager.
 //
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUTargetMachine.h"
 #include "AMDGPU.h"
 #include "AMDGPUAliasAnalysis.h"
-#include "AMDGPUCodeGenPassBuilder.h"
 #include "AMDGPUCtorDtorLowering.h"
 #include "AMDGPUExportClustering.h"
 #include "AMDGPUIGroupLP.h"
@@ -31,7 +32,6 @@
 #include "GCNSchedStrategy.h"
 #include "GCNVOPDUtils.h"
 #include "R600.h"
-#include "R600MachineFunctionInfo.h"
 #include "R600TargetMachine.h"
 #include "SIFixSGPRCopies.h"
 #include "SIMachineFunctionInfo.h"
@@ -40,6 +40,7 @@
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
@@ -64,10 +65,17 @@
 #include "llvm/Transforms/IPO/GlobalDCE.h"
 #include "llvm/Transforms/IPO/Internalize.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Scalar/FlattenCFG.h"
 #include "llvm/Transforms/Scalar/GVN.h"
 #include "llvm/Transforms/Scalar/InferAddressSpaces.h"
+#include "llvm/Transforms/Scalar/Sink.h"
+#include "llvm/Transforms/Scalar/StructurizeCFG.h"
 #include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/FixIrreducible.h"
+#include "llvm/Transforms/Utils/LCSSA.h"
+#include "llvm/Transforms/Utils/LowerSwitch.h"
 #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
+#include "llvm/Transforms/Utils/UnifyLoopExits.h"
 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
 #include <optional>
 
@@ -930,7 +938,7 @@ Error GCNTargetMachine::buildCodeGenPipeline(
 }
 
 //===----------------------------------------------------------------------===//
-// AMDGPU Pass Setup
+// AMDGPU Legacy Pass Setup
 //===----------------------------------------------------------------------===//
 
 std::unique_ptr<CSEConfigBase> llvm::AMDGPUPassConfig::getCSEConfig() const {
@@ -1206,15 +1214,8 @@ AMDGPUPassConfig::createMachineScheduler(MachineSchedContext *C) const {
   return DAG;
 }
 
-MachineFunctionInfo *R600TargetMachine::createMachineFunctionInfo(
-    BumpPtrAllocator &Allocator, const Function &F,
-    const TargetSubtargetInfo *STI) const {
-  return R600MachineFunctionInfo::create<R600MachineFunctionInfo>(
-      Allocator, F, static_cast<const R600Subtarget *>(STI));
-}
-
 //===----------------------------------------------------------------------===//
-// GCN Pass Setup
+// GCN Legacy Pass Setup
 //===----------------------------------------------------------------------===//
 
 ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(
@@ -1751,3 +1752,90 @@ bool GCNTargetMachine::parseMachineFunctionInfo(
 
   return false;
 }
+
+//===----------------------------------------------------------------------===//
+// AMDGPU CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
+    GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
+    PassInstrumentationCallbacks *PIC)
+    : CodeGenPassBuilder(TM, Opts, PIC) {
+  Opt.RequiresCodeGenSCCOrder = true;
+  // Exceptions and StackMaps are not supported, so these passes will never do
+  // anything.
+  // Garbage collection is not supported.
+  disablePass<StackMapLivenessPass, FuncletLayoutPass,
+              ShadowStackGCLoweringPass>();
+}
+
+void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
+  Base::addCodeGenPrepare(addPass);
+
+  // LowerSwitch pass may introduce unreachable blocks that can cause unexpected
+  // behavior for subsequent passes. Placing it here seems better that these
+  // blocks would get cleaned up by UnreachableBlockElim inserted next in the
+  // pass flow.
+  addPass(LowerSwitchPass());
+}
+
+void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
+  const bool LateCFGStructurize = AMDGPUTargetMachine::EnableLateStructurizeCFG;
+  const bool DisableStructurizer = AMDGPUTargetMachine::DisableStructurizer;
+  const bool EnableStructurizerWorkarounds =
+      AMDGPUTargetMachine::EnableStructurizerWorkarounds;
+
+  if (TM.getOptLevel() > CodeGenOptLevel::None)
+    addPass(FlattenCFGPass());
+
+  if (TM.getOptLevel() > CodeGenOptLevel::None)
+    addPass(SinkingPass());
+
+  addPass(AMDGPULateCodeGenPreparePass(TM));
+
+  // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
+  // regions formed by them.
+
+  addPass(AMDGPUUnifyDivergentExitNodesPass());
+
+  if (!LateCFGStructurize && !DisableStructurizer) {
+    if (EnableStructurizerWorkarounds) {
+      addPass(FixIrreduciblePass());
+      addPass(UnifyLoopExitsPass());
+    }
+
+    addPass(StructurizeCFGPass(/*SkipUniformRegions=*/false));
+  }
+
+  addPass(AMDGPUAnnotateUniformValuesPass());
+
+  if (!LateCFGStructurize && !DisableStructurizer) {
+    addPass(SIAnnotateControlFlowPass(TM));
+
+    // TODO: Move this right after structurizeCFG to avoid extra divergence
+    // analysis. This depends on stopping SIAnnotateControlFlow from making
+    // control flow modifications.
+    addPass(AMDGPURewriteUndefForPHIPass());
+  }
+
+  addPass(LCSSAPass());
+
+  if (TM.getOptLevel() > CodeGenOptLevel::Less)
+    addPass(AMDGPUPerfHintAnalysisPass(TM));
+
+  // FIXME: Why isn't this queried as required from AMDGPUISelDAGToDAG, and why
+  // isn't this in addInstSelector?
+  addPass(RequireAnalysisPass<UniformityInfoAnalysis, Function>());
+}
+
+void AMDGPUCodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,
+                                             CreateMCStreamer) const {
+  // TODO: Add AsmPrinter.
+}
+
+Error AMDGPUCodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const {
+  addPass(AMDGPUISelDAGToDAGPass(TM));
+  addPass(SIFixSGPRCopiesPass());
+  addPass(SILowerI1CopiesPass());
+  return Error::success();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 4d39ad2b4150521..576bf40e3328d57 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -16,6 +16,8 @@
 
 #include "GCNSubtarget.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Passes/CodeGenPassBuilder.h"
 #include "llvm/Target/TargetMachine.h"
 #include <optional>
 #include <utility>
@@ -47,7 +49,8 @@ class AMDGPUTargetMachine : public LLVMTargetMachine {
   ~AMDGPUTargetMachine() override;
 
   const TargetSubtargetInfo *getSubtargetImpl() const;
-  const TargetSubtargetInfo *getSubtargetImpl(const Function &) const override = 0;
+  const TargetSubtargetInfo *
+  getSubtargetImpl(const Function &) const override = 0;
 
   TargetLoweringObjectFile *getObjFileLowering() const override {
     return TLOF.get();
@@ -94,9 +97,7 @@ class GCNTargetMachine final : public AMDGPUTargetMachine {
 
   TargetTransformInfo getTargetTransformInfo(const Function &F) const override;
 
-  bool useIPRA() const override {
-    return true;
-  }
+  bool useIPRA() const override { return true; }
 
   Error buildCodeGenPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out,
                              raw_pwrite_stream *DwoOut,
@@ -120,7 +121,7 @@ class GCNTargetMachine final : public AMDGPUTargetMachine {
 };
 
 //===----------------------------------------------------------------------===//
-// AMDGPU Pass Setup
+// AMDGPU Pass Setup - For Legacy Pass Manager.
 //===----------------------------------------------------------------------===//
 
 class AMDGPUPassConfig : public TargetPassConfig {
@@ -158,6 +159,25 @@ class AMDGPUPassConfig : public TargetPassConfig {
   }
 };
 
+//===----------------------------------------------------------------------===//
+// AMDGPU CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+class AMDGPUCodeGenPassBuilder
+    : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {
+  using Base = CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine>;
+
+public:
+  AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
+                           const CGPassBuilderOption &Opts,
+                           PassInstrumentationCallbacks *PIC);
+
+  void addCodeGenPrepare(AddIRPass &) const;
+  void addPreISel(AddIRPass &addPass) const;
+  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
+  Error addInstSelector(AddMachinePass &) const;
+};
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETMACHINE_H
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 671caf8484cd971..c91584ae8ce59cd 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -51,7 +51,6 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUAtomicOptimizer.cpp
   AMDGPUAttributor.cpp
   AMDGPUCallLowering.cpp
-  AMDGPUCodeGenPassBuilder.cpp
   AMDGPUCodeGenPrepare.cpp
   AMDGPUCombinerHelper.cpp
   AMDGPUCtorDtorLowering.cpp
@@ -122,7 +121,6 @@ add_llvm_target(AMDGPUCodeGen
   GCNVOPDUtils.cpp
   R600AsmPrinter.cpp
   R600ClauseMergePass.cpp
-  R600CodeGenPassBuilder.cpp
   R600ControlFlowFinalizer.cpp
   R600EmitClauseMarkers.cpp
   R600ExpandSpecialInstrs.cpp
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index def89c785b85523..902f51ae358d59c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -1289,6 +1289,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   /// and STOREcnt rather than VMcnt, LGKMcnt and VScnt respectively.
   bool hasExtendedWaitCounts() const { return getGeneration() >= GFX12; }
 
+  /// \returns true if inline constants are not supported for F16 pseudo
+  /// scalar transcendentals.
+  bool hasNoF16PseudoScalarTransInlineConstants() const {
+    return getGeneration() == GFX12;
+  }
+
   /// \returns The maximum number of instructions that can be enclosed in an
   /// S_CLAUSE on the given subtarget, or 0 for targets that do not support that
   /// instruction.
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
index 201cc8d01e2d4b4..3c9f6d2938075b8 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
@@ -34,7 +34,6 @@ AMDGPUMCAsmInfo::AMDGPUMCAsmInfo(const Triple &TT,
   UsesELFSectionDirectiveForBSS = true;
 
   //===--- Global Variable Emission Directives --------------------------===//
-  HasAggressiveSymbolFolding = true;
   COMMDirectiveAlignmentIsInBytes = false;
   HasNoDeadStrip = true;
   //===--- Dwarf Emission Directives -----------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
index b2a9667f7c23ea2..4fbd7d0f889457d 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
@@ -10,12 +10,14 @@
 #include "GCNSubtarget.h"
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/IR/Function.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/KnownBits.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -303,3 +305,375 @@ const AMDGPUMCExpr *AMDGPUMCExpr::createOccupancy(unsigned InitOcc,
                  CreateExpr(InitOcc), NumSGPRs, NumVGPRs},
                 Ctx);
 }
+
+static KnownBits fromOptionalToKnownBits(std::optional<bool> CompareResult) {
+  static constexpr unsigned BitWidth = 64;
+  const APInt True(BitWidth, 1);
+  const APInt False(BitWidth, 0);
+  if (CompareResult) {
+    return *CompareResult ? KnownBits::makeConstant(True)
+                          : KnownBits::makeConstant(False);
+  }
+
+  KnownBits UnknownBool(/*BitWidth=*/1);
+  return UnknownBool.zext(BitWidth);
+}
+
+using KnownBitsMap = DenseMap<const MCExpr *, KnownBits>;
+static void knownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                               unsigned Depth = 0);
+
+static void binaryOpKnownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                                       unsigned Depth) {
+  static constexpr unsigned BitWidth = 64;
+  const MCBinaryExpr *BExpr = cast<MCBinaryExpr>(Expr);
+  const MCExpr *LHS = BExpr->getLHS();
+  const MCExpr *RHS = BExpr->getRHS();
+
+  knownBitsMapHelper(LHS, KBM, Depth + 1);
+  knownBitsMapHelper(RHS, KBM, Depth + 1);
+  KnownBits LHSKnown = KBM[LHS];
+  KnownBits RHSKnown = KBM[RHS];
+
+  switch (BExpr->getOpcode()) {
+  default:
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  case MCBinaryExpr::Opcode::Add:
+    KBM[Expr] = KnownBits::add(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::And:
+    KBM[Expr] = LHSKnown & RHSKnown;
+    return;
+  case MCBinaryExpr::Opcode::Div:
+    KBM[Expr] = KnownBits::sdiv(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::EQ: {
+    std::optional<bool> CompareRes = KnownBits::eq(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::NE: {
+    std::optional<bool> CompareRes = KnownBits::ne(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::GT: {
+    std::optional<bool> CompareRes = KnownBits::sgt(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::GTE: {
+    std::optional<bool> CompareRes = KnownBits::sge(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::LAnd: {
+    std::optional<bool> CompareRes;
+    const APInt False(BitWidth, 0);
+    std::optional<bool> LHSBool =
+        KnownBits::ne(LHSKnown, KnownBits::makeConstant(False));
+    std::optional<bool> RHSBool =
+        KnownBits::ne(RHSKnown, KnownBits::makeConstant(False));
+    if (LHSBool && RHSBool)
+      CompareRes = *LHSBool && *RHSBool;
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::LOr: {
+    const APInt False(BitWidth, 0);
+    KnownBits Bits = LHSKnown | RHSKnown;
+    std::optional<bool> CompareRes =
+        KnownBits::ne(Bits, KnownBits::makeConstant(False));
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::LT: {
+    std::optional<bool> CompareRes = KnownBits::slt(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::LTE: {
+    std::optional<bool> CompareRes = KnownBits::sle(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::Mod:
+    KBM[Expr] = KnownBits::srem(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::Mul:
+    KBM[Expr] = KnownBits::mul(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::Or:
+    KBM[Expr] = LHSKnown | RHSKnown;
+    return;
+  case MCBinaryExpr::Opcode::Shl:
+    KBM[Expr] = KnownBits::shl(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::AShr:
+    KBM[Expr] = KnownBits::ashr(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::LShr:
+    KBM[Expr] = KnownBits::lshr(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::Sub:
+    KBM[Expr] = KnownBits::sub(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::Xor:
+    KBM[Expr] = LHSKnown ^ RHSKnown;
+    return;
+  }
+}
+
+static void unaryOpKnownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                                      unsigned Depth) {
+  static constexpr unsigned BitWidth = 64;
+  const MCUnaryExpr *UExpr = cast<MCUnaryExpr>(Expr);
+  knownBitsMapHelper(UExpr->getSubExpr(), KBM, Depth + 1);
+  KnownBits KB = KBM[UExpr->getSubExpr()];
+
+  switch (UExpr->getOpcode()) {
+  default:
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  case MCUnaryExpr::Opcode::Minus: {
+    KB.makeNegative();
+    KBM[Expr] = KB;
+    return;
+  }
+  case MCUnaryExpr::Opcode::Not: {
+    KnownBits AllOnes(BitWidth);
+    AllOnes.setAllOnes();
+    KBM[Expr] = KB ^ AllOnes;
+    return;
+  }
+  case MCUnaryExpr::Opcode::Plus: {
+    KB.makeNonNegative();
+    KBM[Expr] = KB;
+    return;
+  }
+  }
+}
+
+static void targetOpKnownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                                       unsigned Depth) {
+  static constexpr unsigned BitWidth = 64;
+  const AMDGPUMCExpr *AGVK = cast<AMDGPUMCExpr>(Expr);
+
+  switch (AGVK->getKind()) {
+  default:
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  case AMDGPUMCExpr::VariantKind::AGVK_Or: {
+    knownBitsMapHelper(AGVK->getSubExpr(0), KBM, Depth + 1);
+    KnownBits KB = KBM[AGVK->getSubExpr(0)];
+    for (const MCExpr *Arg : AGVK->getArgs()) {
+      knownBitsMapHelper(Arg, KBM, Depth + 1);
+      KB |= KBM[Arg];
+    }
+    KBM[Expr] = KB;
+    return;
+  }
+  case AMDGPUMCExpr::VariantKind::AGVK_Max: {
+    knownBitsMapHelper(AGVK->getSubExpr(0), KBM, Depth + 1);
+    KnownBits KB = KBM[AGVK->getSubExpr(0)];
+    for (const MCExpr *Arg : AGVK->getArgs()) {
+      knownBitsMapHelper(Arg, KBM, Depth + 1);
+      KB = KnownBits::umax(KB, KBM[Arg]);
+    }
+    KBM[Expr] = KB;
+    return;
+  }
+  case AMDGPUMCExpr::VariantKind::AGVK_ExtraSGPRs:
+  case AMDGPUMCExpr::VariantKind::AGVK_TotalNumVGPRs:
+  case AMDGPUMCExpr::VariantKind::AGVK_AlignTo:
+  case AMDGPUMCExpr::VariantKind::AGVK_Occupancy: {
+    int64_t Val;
+    if (AGVK->evaluateAsAbsolute(Val)) {
+      APInt APValue(BitWidth, Val);
+      KBM[Expr] = KnownBits::makeConstant(APValue);
+      return;
+    }
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  }
+  }
+}
+
+static void knownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                               unsigned Depth) {
+  static constexpr unsigned BitWidth = 64;
+
+  int64_t Val;
+  if (Expr->evaluateAsAbsolute(Val)) {
+    APInt APValue(BitWidth, Val, /*isSigned=*/true);
+    KBM[Expr] = KnownBits::makeConstant(APValue);
+    return;
+  }
+
+  if (Depth == 16) {
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  }
+
+  switch (Expr->getKind()) {
+  case MCExpr::ExprKind::Binary: {
+    binaryOpKnownBitsMapHelper(Expr, KBM, Depth);
+    return;
+  }
+  case MCExpr::ExprKind::Constant: {
+    const MCConstantExpr *CE = cast<MCConstantExpr>(Expr);
+    APInt APValue(BitWidth, CE->getValue(), /*isSigned=*/true);
+    KBM[Expr] = KnownBits::makeConstant(APValue);
+    return;
+  }
+  case MCExpr::ExprKind::SymbolRef: {
+    const MCSymbolRefExpr *RExpr = cast<MCSymbolRefExpr>(Expr);
+    const MCSymbol &Sym = RExpr->getSymbol();
+    if (!Sym.isVariable()) {
+      KBM[Expr] = KnownBits(BitWidth);
+      return;
+    }
+
+    // Variable value retrieval is not for actual use but only for knownbits
+    // analysis.
+    knownBitsMapHelper(Sym.getVariableValue(/*SetUsed=*/false), KBM, Depth + 1);
+    KBM[Expr] = KBM[Sym.getVariableValue(/*SetUsed=*/false)];
+    return;
+  }
+  case MCExpr::ExprKind::Unary: {
+    unaryOpKnownBitsMapHelper(Expr, KBM, Depth);
+    return;
+  }
+  case MCExpr::ExprKind::Target: {
+    targetOpKnownBitsMapHelper(Expr, KBM, Depth);
+    return;
+  }
+  }
+}
+
+static const MCExpr *tryFoldHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                                   MCContext &Ctx) {
+  if (!KBM.count(Expr))
+    return Expr;
+
+  auto ValueCheckKnownBits = [](KnownBits &KB, unsigned Value) -> bool {
+    if (!KB.isConstant())
+      return false;
+
+    return Value == KB.getConstant();
+  };
+
+  if (Expr->getKind() == MCExpr::ExprKind::Constant)
+    return Expr;
+
+  // Resolving unary operations to constants may make the value more ambiguous.
+  // For example, `~62` becomes `-63`; however, to me it's more ambiguous if a
+  // bit mask value is represented through a negative number.
+  if (Expr->getKind() != MCExpr::ExprKind::Unary) {
+    if (KBM[Expr].isConstant()) {
+      APInt ConstVal = KBM[Expr].getConstant();
+      return MCConstantExpr::create(ConstVal.getSExtValue(), Ctx);
+    }
+
+    int64_t EvalValue;
+    if (Expr->evaluateAsAbsolute(EvalValue))
+      return MCConstantExpr::create(EvalValue, Ctx);
+  }
+
+  switch (Expr->getKind()) {
+  default:
+    return Expr;
+  case MCExpr::ExprKind::Binary: {
+    const MCBinaryExpr *BExpr = cast<MCBinaryExpr>(Expr);
+    const MCExpr *LHS = BExpr->getLHS();
+    const MCExpr *RHS = BExpr->getRHS();
+
+    switch (BExpr->getOpcode()) {
+    default:
+      return Expr;
+    case MCBinaryExpr::Opcode::Sub: {
+      if (ValueCheckKnownBits(KBM[RHS], 0))
+        return tryFoldHelper(LHS, KBM, Ctx);
+      break;
+    }
+    case MCBinaryExpr::Opcode::Add:
+    case MCBinaryExpr::Opcode::Or: {
+      if (ValueCheckKnownBits(KBM[LHS], 0))
+        return tryFoldHelper(RHS, KBM, Ctx);
+      if (ValueCheckKnownBits(KBM[RHS], 0))
+        return tryFoldHelper(LHS, KBM, Ctx);
+      break;
+    }
+    case MCBinaryExpr::Opcode::Mul: {
+      if (ValueCheckKnownBits(KBM[LHS], 1))
+        return tryFoldHelper(RHS, KBM, Ctx);
+      if (ValueCheckKnownBits(KBM[RHS], 1))
+        return tryFoldHelper(LHS, KBM, Ctx);
+      break;
+    }
+    case MCBinaryExpr::Opcode::Shl:
+    case MCBinaryExpr::Opcode::AShr:
+    case MCBinaryExpr::Opcode::LShr: {
+      if (ValueCheckKnownBits(KBM[RHS], 0))
+        return tryFoldHelper(LHS, KBM, Ctx);
+      if (ValueCheckKnownBits(KBM[LHS], 0))
+        return MCConstantExpr::create(0, Ctx);
+      break;
+    }
+    case MCBinaryExpr::Opcode::And: {
+      if (ValueCheckKnownBits(KBM[LHS], 0) || ValueCheckKnownBits(KBM[RHS], 0))
+        return MCConstantExpr::create(0, Ctx);
+      break;
+    }
+    }
+    const MCExpr *NewLHS = tryFoldHelper(LHS, KBM, Ctx);
+    const MCExpr *NewRHS = tryFoldHelper(RHS, KBM, Ctx);
+    if (NewLHS != LHS || NewRHS != RHS)
+      return MCBinaryExpr::create(BExpr->getOpcode(), NewLHS, NewRHS, Ctx,
+                                  BExpr->getLoc());
+    return Expr;
+  }
+  case MCExpr::ExprKind::Unary: {
+    const MCUnaryExpr *UExpr = cast<MCUnaryExpr>(Expr);
+    const MCExpr *SubExpr = UExpr->getSubExpr();
+    const MCExpr *NewSubExpr = tryFoldHelper(SubExpr, KBM, Ctx);
+    if (SubExpr != NewSubExpr)
+      return MCUnaryExpr::create(UExpr->getOpcode(), NewSubExpr, Ctx,
+                                 UExpr->getLoc());
+    return Expr;
+  }
+  case MCExpr::ExprKind::Target: {
+    const AMDGPUMCExpr *AGVK = cast<AMDGPUMCExpr>(Expr);
+    SmallVector<const MCExpr *, 8> NewArgs;
+    bool Changed = false;
+    for (const MCExpr *Arg : AGVK->getArgs()) {
+      const MCExpr *NewArg = tryFoldHelper(Arg, KBM, Ctx);
+      NewArgs.push_back(NewArg);
+      Changed |= Arg != NewArg;
+    }
+    return Changed ? AMDGPUMCExpr::create(AGVK->getKind(), NewArgs, Ctx) : Expr;
+  }
+  }
+  return Expr;
+}
+
+const MCExpr *llvm::AMDGPU::foldAMDGPUMCExpr(const MCExpr *Expr,
+                                             MCContext &Ctx) {
+  KnownBitsMap KBM;
+  knownBitsMapHelper(Expr, KBM);
+  const MCExpr *NewExpr = tryFoldHelper(Expr, KBM, Ctx);
+
+  return Expr != NewExpr ? NewExpr : Expr;
+}
+
+void llvm::AMDGPU::printAMDGPUMCExpr(const MCExpr *Expr, raw_ostream &OS,
+                                     const MCAsmInfo *MAI) {
+  int64_t Val;
+  if (Expr->evaluateAsAbsolute(Val)) {
+    OS << Val;
+    return;
+  }
+
+  Expr->print(OS, MAI);
+}
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
index 970802dab69b60a..a16843f404b8f62 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
@@ -90,6 +90,7 @@ class AMDGPUMCExpr : public MCTargetExpr {
                                              const GCNSubtarget &STM,
                                              MCContext &Ctx);
 
+  ArrayRef<const MCExpr *> getArgs() const { return Args; }
   VariantKind getKind() const { return Kind; }
   const MCExpr *getSubExpr(size_t Index) const;
 
@@ -105,6 +106,18 @@ class AMDGPUMCExpr : public MCTargetExpr {
   }
 };
 
+namespace AMDGPU {
+// Tries to leverage KnownBits for MCExprs to reduce and limit any composed
+// MCExprs printing. E.g., for an expression such as
+// ((unevaluatable_sym | 1) & 1) won't evaluate due to unevaluatable_sym and
+// would verbosely print the full expression; however, KnownBits should deduce
+// the value to be 1. Particularly useful for AMDGPU metadata MCExprs.
+void printAMDGPUMCExpr(const MCExpr *Expr, raw_ostream &OS,
+                       const MCAsmInfo *MAI);
+
+const MCExpr *foldAMDGPUMCExpr(const MCExpr *Expr, MCContext &Ctx);
+
+} // end namespace AMDGPU
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCEXPR_H
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 8c78db8e83c42d6..73d466abc66f7b7 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUTargetStreamer.h"
+#include "AMDGPUMCExpr.h"
 #include "AMDGPUMCKernelDescriptor.h"
 #include "AMDGPUPTNote.h"
 #include "Utils/AMDGPUBaseInfo.h"
@@ -244,8 +245,13 @@ void AMDGPUTargetAsmStreamer::EmitDirectiveAMDHSACodeObjectVersion(
 }
 
 void AMDGPUTargetAsmStreamer::EmitAMDKernelCodeT(AMDGPUMCKernelCodeT &Header) {
+  auto FoldAndPrint = [&](const MCExpr *Expr, raw_ostream &OS,
+                          const MCAsmInfo *MAI) {
+    printAMDGPUMCExpr(foldAMDGPUMCExpr(Expr, getContext()), OS, MAI);
+  };
+
   OS << "\t.amd_kernel_code_t\n";
-  Header.EmitKernelCodeT(OS, getContext());
+  Header.EmitKernelCodeT(OS, getContext(), FoldAndPrint);
   OS << "\t.end_amd_kernel_code_t\n";
 }
 
@@ -329,24 +335,17 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
 
   auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask,
                         StringRef Directive) {
-    int64_t IVal;
     OS << "\t\t" << Directive << ' ';
-    const MCExpr *pgm_rsrc1_bits =
+    const MCExpr *ShiftedAndMaskedExpr =
         MCKernelDescriptor::bits_get(Expr, Shift, Mask, getContext());
-    if (pgm_rsrc1_bits->evaluateAsAbsolute(IVal))
-      OS << static_cast<uint64_t>(IVal);
-    else
-      pgm_rsrc1_bits->print(OS, MAI);
+    const MCExpr *New = foldAMDGPUMCExpr(ShiftedAndMaskedExpr, getContext());
+    printAMDGPUMCExpr(New, OS, MAI);
     OS << '\n';
   };
 
   auto EmitMCExpr = [&](const MCExpr *Value) {
-    int64_t evaluatableValue;
-    if (Value->evaluateAsAbsolute(evaluatableValue)) {
-      OS << static_cast<uint64_t>(evaluatableValue);
-    } else {
-      Value->print(OS, MAI);
-    }
+    const MCExpr *NewExpr = foldAMDGPUMCExpr(Value, getContext());
+    printAMDGPUMCExpr(NewExpr, OS, MAI);
   };
 
   OS << "\t\t.amdhsa_group_segment_fixed_size ";
@@ -462,7 +461,8 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     accum_bits = MCBinaryExpr::createMul(
         accum_bits, MCConstantExpr::create(4, getContext()), getContext());
     OS << "\t\t.amdhsa_accum_offset ";
-    EmitMCExpr(accum_bits);
+    const MCExpr *New = foldAMDGPUMCExpr(accum_bits, getContext());
+    printAMDGPUMCExpr(New, OS, MAI);
     OS << '\n';
   }
 
diff --git a/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.cpp b/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.cpp
deleted file mode 100644
index a57b3aa0adb158a..000000000000000
--- a/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.cpp
+++ /dev/null
@@ -1,33 +0,0 @@
-//===-- R600CodeGenPassBuilder.cpp ------ Build R600 CodeGen pipeline -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "R600CodeGenPassBuilder.h"
-#include "R600TargetMachine.h"
-
-using namespace llvm;
-
-R600CodeGenPassBuilder::R600CodeGenPassBuilder(
-    R600TargetMachine &TM, const CGPassBuilderOption &Opts,
-    PassInstrumentationCallbacks *PIC)
-    : CodeGenPassBuilder(TM, Opts, PIC) {
-  Opt.RequiresCodeGenSCCOrder = true;
-}
-
-void R600CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
-  // TODO: Add passes pre instruction selection.
-}
-
-void R600CodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,
-                                           CreateMCStreamer) const {
-  // TODO: Add AsmPrinter.
-}
-
-Error R600CodeGenPassBuilder::addInstSelector(AddMachinePass &) const {
-  // TODO: Add instruction selector.
-  return Error::success();
-}
diff --git a/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.h b/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.h
deleted file mode 100644
index be7c935c094d9f0..000000000000000
--- a/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.h
+++ /dev/null
@@ -1,32 +0,0 @@
-//===-- R600CodeGenPassBuilder.h -- Build R600 CodeGen pipeline -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_AMDGPU_R600CODEGENPASSBUILDER_H
-#define LLVM_LIB_TARGET_AMDGPU_R600CODEGENPASSBUILDER_H
-
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/Passes/CodeGenPassBuilder.h"
-
-namespace llvm {
-
-class R600TargetMachine;
-
-class R600CodeGenPassBuilder
-    : public CodeGenPassBuilder<R600CodeGenPassBuilder, R600TargetMachine> {
-public:
-  R600CodeGenPassBuilder(R600TargetMachine &TM, const CGPassBuilderOption &Opts,
-                         PassInstrumentationCallbacks *PIC);
-
-  void addPreISel(AddIRPass &addPass) const;
-  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
-  Error addInstSelector(AddMachinePass &) const;
-};
-
-} // namespace llvm
-
-#endif // LLVM_LIB_TARGET_AMDGPU_R600CODEGENPASSBUILDER_H
diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
index c79688bf038be30..7e4d9d21a0b3972 100644
--- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -15,7 +15,6 @@
 #include "AMDGPU.h"
 #include "MCTargetDesc/R600MCTargetDesc.h"
 #include "R600Defines.h"
-#include "R600InstrInfo.h"
 #include "R600MachineFunctionInfo.h"
 #include "R600Subtarget.h"
 #include "R600TargetMachine.h"
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
index c550cfaf06c1002..a1a60b8bdfa9ee1 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
@@ -7,15 +7,16 @@
 //===----------------------------------------------------------------------===//
 //
 /// \file
-/// The AMDGPU-R600 target machine contains all of the hardware specific
-/// information  needed to emit code for R600 GPUs.
+/// This file contains both AMDGPU-R600 target machine and the CodeGen pass
+/// builder. The target machine contains all of the hardware specific
+/// information needed to emit code for R600 GPUs and the CodeGen pass builder
+/// handles the pass pipeline for new pass manager.
 //
 //===----------------------------------------------------------------------===//
 
 #include "R600TargetMachine.h"
-#include "AMDGPUTargetMachine.h"
 #include "R600.h"
-#include "R600CodeGenPassBuilder.h"
+#include "R600MachineFunctionInfo.h"
 #include "R600MachineScheduler.h"
 #include "R600TargetTransformInfo.h"
 #include "llvm/Transforms/Scalar.h"
@@ -153,3 +154,35 @@ Error R600TargetMachine::buildCodeGenPipeline(
   R600CodeGenPassBuilder CGPB(*this, Opts, PIC);
   return CGPB.buildPipeline(MPM, Out, DwoOut, FileType);
 }
+
+MachineFunctionInfo *R600TargetMachine::createMachineFunctionInfo(
+    BumpPtrAllocator &Allocator, const Function &F,
+    const TargetSubtargetInfo *STI) const {
+  return R600MachineFunctionInfo::create<R600MachineFunctionInfo>(
+      Allocator, F, static_cast<const R600Subtarget *>(STI));
+}
+
+//===----------------------------------------------------------------------===//
+// R600 CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+R600CodeGenPassBuilder::R600CodeGenPassBuilder(
+    R600TargetMachine &TM, const CGPassBuilderOption &Opts,
+    PassInstrumentationCallbacks *PIC)
+    : CodeGenPassBuilder(TM, Opts, PIC) {
+  Opt.RequiresCodeGenSCCOrder = true;
+}
+
+void R600CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
+  // TODO: Add passes pre instruction selection.
+}
+
+void R600CodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,
+                                           CreateMCStreamer) const {
+  // TODO: Add AsmPrinter.
+}
+
+Error R600CodeGenPassBuilder::addInstSelector(AddMachinePass &) const {
+  // TODO: Add instruction selector.
+  return Error::success();
+}
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.h b/llvm/lib/Target/AMDGPU/R600TargetMachine.h
index 29e370edef2c67b..b7f123a07a9c1de 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.h
@@ -55,6 +55,21 @@ class R600TargetMachine final : public AMDGPUTargetMachine {
                             const TargetSubtargetInfo *STI) const override;
 };
 
+//===----------------------------------------------------------------------===//
+// R600 CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+class R600CodeGenPassBuilder
+    : public CodeGenPassBuilder<R600CodeGenPassBuilder, R600TargetMachine> {
+public:
+  R600CodeGenPassBuilder(R600TargetMachine &TM, const CGPassBuilderOption &Opts,
+                         PassInstrumentationCallbacks *PIC);
+
+  void addPreISel(AddIRPass &addPass) const;
+  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
+  Error addInstSelector(AddMachinePass &) const;
+};
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_R600TARGETMACHINE_H
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 25fee559faa29ca..ae55b56fbf43fb7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5804,24 +5804,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FP_ROUND:
   case ISD::STRICT_FP_ROUND:
     return lowerFP_ROUND(Op, DAG);
-  case ISD::FPTRUNC_ROUND: {
-    unsigned Opc;
-    SDLoc DL(Op);
-
-    if (Op.getOperand(0)->getValueType(0) != MVT::f32)
-      return SDValue();
-
-    // Get the rounding mode from the last operand
-    int RoundMode = Op.getConstantOperandVal(1);
-    if (RoundMode == (int)RoundingMode::TowardPositive)
-      Opc = AMDGPUISD::FPTRUNC_ROUND_UPWARD;
-    else if (RoundMode == (int)RoundingMode::TowardNegative)
-      Opc = AMDGPUISD::FPTRUNC_ROUND_DOWNWARD;
-    else
-      return SDValue();
-
-    return DAG.getNode(Opc, DL, Op.getNode()->getVTList(), Op->getOperand(0));
-  }
+  case ISD::FPTRUNC_ROUND:
+    return lowerFPTRUNC_ROUND(Op, DAG);
   case ISD::TRAP:
     return lowerTRAP(Op, DAG);
   case ISD::DEBUGTRAP:
@@ -6671,6 +6655,30 @@ SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG,
                 DAG.getTargetConstant(0, DL, MVT::i32));
 }
 
+SDValue SITargetLowering::lowerFPTRUNC_ROUND(SDValue Op,
+                                             SelectionDAG &DAG) const {
+  if (Op.getOperand(0)->getValueType(0) != MVT::f32)
+    return SDValue();
+
+  // Only support towardzero, tonearest, upward and downward.
+  int RoundMode = Op.getConstantOperandVal(1);
+  if (RoundMode != (int)RoundingMode::TowardZero &&
+      RoundMode != (int)RoundingMode::NearestTiesToEven &&
+      RoundMode != (int)RoundingMode::TowardPositive &&
+      RoundMode != (int)RoundingMode::TowardNegative)
+    return SDValue();
+
+  // "round.towardzero" -> TowardZero 0        -> FP_ROUND_ROUND_TO_ZERO 3
+  // "round.tonearest"  -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
+  // "round.upward"     -> TowardPositive 2    -> FP_ROUND_ROUND_TO_INF 1
+  // "round.downward    -> TowardNegative 3    -> FP_ROUND_ROUND_TO_NEGINF 2
+  unsigned HW_Mode = (RoundMode + 3) % 4;
+  SDLoc DL(Op);
+  SDValue RoundFlag = DAG.getTargetConstant(HW_Mode, DL, MVT::i32);
+  return DAG.getNode(AMDGPUISD::FPTRUNC_ROUND, DL, Op.getNode()->getVTList(),
+                     Op->getOperand(0), RoundFlag);
+}
+
 SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
   assert(Op.getValueType() == MVT::f16 &&
          "Do not know how to custom lower FP_ROUND for non-f16 type");
@@ -15750,16 +15758,12 @@ void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     case Intrinsic::amdgcn_mbcnt_hi: {
       const GCNSubtarget &ST =
           DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
-      // These return at most the (wavefront size - 1) + src1
-      // As long as src1 is an immediate we can calc known bits
-      KnownBits Src1Known = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
-      unsigned Src1ValBits = Src1Known.countMaxActiveBits();
-      unsigned MaxActiveBits = std::max(Src1ValBits, ST.getWavefrontSizeLog2());
-      // Cater for potential carry
-      MaxActiveBits += Src1ValBits ? 1 : 0;
-      unsigned Size = Op.getValueType().getSizeInBits();
-      if (MaxActiveBits < Size)
-        Known.Zero.setHighBits(Size - MaxActiveBits);
+      // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
+      // most 31 + src1.
+      Known.Zero.setBitsFrom(
+          IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
+      KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
+      Known = KnownBits::add(Known, Known2);
       return;
     }
     }
@@ -15794,7 +15798,8 @@ void SITargetLowering::computeKnownBitsForTargetInstr(
   switch (MI->getOpcode()) {
   case AMDGPU::G_INTRINSIC:
   case AMDGPU::G_INTRINSIC_CONVERGENT: {
-    switch (cast<GIntrinsic>(MI)->getIntrinsicID()) {
+    Intrinsic::ID IID = cast<GIntrinsic>(MI)->getIntrinsicID();
+    switch (IID) {
     case Intrinsic::amdgcn_workitem_id_x:
       knownBitsForWorkitemID(*getSubtarget(), KB, Known, 0);
       break;
@@ -15806,9 +15811,15 @@ void SITargetLowering::computeKnownBitsForTargetInstr(
       break;
     case Intrinsic::amdgcn_mbcnt_lo:
     case Intrinsic::amdgcn_mbcnt_hi: {
-      // These return at most the wavefront size - 1.
-      unsigned Size = MRI.getType(R).getSizeInBits();
-      Known.Zero.setHighBits(Size - getSubtarget()->getWavefrontSizeLog2());
+      // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
+      // most 31 + src1.
+      Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
+                                 ? getSubtarget()->getWavefrontSizeLog2()
+                                 : 5);
+      KnownBits Known2;
+      KB.computeKnownBitsImpl(MI->getOperand(3).getReg(), Known2, DemandedElts,
+                              Depth + 1);
+      Known = KnownBits::add(Known, Known2);
       break;
     }
     case Intrinsic::amdgcn_groupstaticsize: {
@@ -16093,7 +16104,7 @@ static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW) {
          << " operation at memory scope " << MemScope;
 }
 
-static bool isHalf2OrBFloat2(Type *Ty) {
+static bool isV2F16OrV2BF16(Type *Ty) {
   if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
     Type *EltTy = VT->getElementType();
     return VT->getNumElements() == 2 &&
@@ -16103,12 +16114,12 @@ static bool isHalf2OrBFloat2(Type *Ty) {
   return false;
 }
 
-static bool isHalf2(Type *Ty) {
+static bool isV2F16(Type *Ty) {
   FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty);
   return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
 }
 
-static bool isBFloat2(Type *Ty) {
+static bool isV2BF16(Type *Ty) {
   FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty);
   return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
 }
@@ -16248,7 +16259,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
                                                  : AtomicExpansionKind::CmpXChg;
       }
 
-      if (Subtarget->hasAtomicDsPkAdd16Insts() && isHalf2OrBFloat2(Ty))
+      if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
         return AtomicExpansionKind::None;
 
       return AtomicExpansionKind::CmpXChg;
@@ -16273,24 +16284,24 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
     if (globalMemoryFPAtomicIsLegal(*Subtarget, RMW, HasSystemScope)) {
       if (AS == AMDGPUAS::FLAT_ADDRESS) {
         // gfx940, gfx12
-        if (Subtarget->hasAtomicFlatPkAdd16Insts() && isHalf2OrBFloat2(Ty))
+        if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
       } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
         // gfx90a, gfx940, gfx12
-        if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isHalf2(Ty))
+        if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
 
         // gfx940, gfx12
-        if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isBFloat2(Ty))
+        if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
       } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
         // gfx90a, gfx940, gfx12
-        if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isHalf2(Ty))
+        if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
 
         // While gfx90a/gfx940 supports v2bf16 for global/flat, it does not for
         // buffer. gfx12 does have the buffer version.
-        if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isBFloat2(Ty))
+        if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
       }
 
@@ -16311,7 +16322,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
           // gfx908
           if (RMW->use_empty() &&
               Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
-              isHalf2(Ty))
+              isV2F16(Ty))
             return ReportUnsafeHWInst(AtomicExpansionKind::None);
         }
       }
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 1f198a92c0fa6a3..eed4b3e79cdeee8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -145,6 +145,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
 
   /// Custom lowering for ISD::FP_ROUND for MVT::f16.
   SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerFPTRUNC_ROUND(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerMUL(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 9147242046cedad..6dce41d1605fa41 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5776,6 +5776,10 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
           return false;
       }
     }
+  } else if (ST.hasNoF16PseudoScalarTransInlineConstants() && !MO->isReg() &&
+             isF16PseudoScalarTrans(MI.getOpcode()) &&
+             isInlineConstant(*MO, OpInfo)) {
+    return false;
   }
 
   if (MO->isReg()) {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 1712dfe8d406ccb..91855fb14f6f371 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -946,6 +946,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
            Opcode == AMDGPU::DS_GWS_BARRIER;
   }
 
+  static bool isF16PseudoScalarTrans(unsigned Opcode) {
+    return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
+           Opcode == AMDGPU::V_S_LOG_F16_e64 ||
+           Opcode == AMDGPU::V_S_RCP_F16_e64 ||
+           Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
+           Opcode == AMDGPU::V_S_SQRT_F16_e64;
+  }
+
   static bool doesNotReadTiedSource(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::TiedSourceNotRead;
   }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index e99b43afd1c3a28..85281713e22b1f8 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -304,13 +304,11 @@ def SIdenorm_mode : SDNode<"AMDGPUISD::DENORM_MODE",
   [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]
 >;
 
-def SIfptrunc_round_upward : SDNode<"AMDGPUISD::FPTRUNC_ROUND_UPWARD",
-  SDTFPRoundOp
->;
+def SDTFPRoundModeOp  : SDTypeProfile<1, 2, [
+  SDTCisFP<0>, SDTCisFP<1>, SDTCisInt<2>, SDTCisOpSmallerThanOp<0, 1>, SDTCisSameNumEltsAs<0, 1>
+]>;
 
-def SIfptrunc_round_downward : SDNode<"AMDGPUISD::FPTRUNC_ROUND_DOWNWARD",
-  SDTFPRoundOp
->;
+def SIfptrunc_round : SDNode<"AMDGPUISD::FPTRUNC_ROUND", SDTFPRoundModeOp>;
 
 //===----------------------------------------------------------------------===//
 // ValueType helpers
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 5a139d1cf8d8255..e7831d00a3a4a89 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -225,20 +225,12 @@ def S_INVERSE_BALLOT_U64 : SPseudoInstSI<
 >;
 } // End usesCustomInserter = 1
 
-// Pseudo instructions used for @llvm.fptrunc.round upward
-// and @llvm.fptrunc.round downward.
-// These intrinsics will be legalized to G_FPTRUNC_ROUND_UPWARD
-// and G_FPTRUNC_ROUND_DOWNWARD before being lowered to
-// FPTRUNC_UPWARD_PSEUDO and FPTRUNC_DOWNWARD_PSEUDO.
-// The final codegen is done in the ModeRegister pass.
+// Pseudo instructions used for @llvm.fptrunc.round. The final codegen is done
+// in the ModeRegister pass.
 let Uses = [MODE, EXEC] in {
-def FPTRUNC_UPWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
-  (ins VGPR_32:$src0),
-  [(set f16:$vdst, (SIfptrunc_round_upward f32:$src0))]>;
-
-def FPTRUNC_DOWNWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
-  (ins VGPR_32:$src0),
-  [(set f16:$vdst, (SIfptrunc_round_downward f32:$src0))]>;
+def FPTRUNC_ROUND_F16_F32_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
+  (ins VGPR_32:$src0, i32imm:$round),
+  [(set f16:$vdst, (SIfptrunc_round f32:$src0, i32:$round))]>;
 } // End Uses = [MODE, EXEC]
 
 // Invert the exec mask and overwrite the inactive lanes of dst with inactive,
@@ -4072,15 +4064,9 @@ def G_SI_CALL : AMDGPUGenericInstruction {
   let isConvergent = 1;
 }
 
-def G_FPTRUNC_ROUND_UPWARD : AMDGPUGenericInstruction {
-  let OutOperandList = (outs type0:$vdst);
-  let InOperandList = (ins type1:$src0);
-  let hasSideEffects = 0;
-}
-
-def G_FPTRUNC_ROUND_DOWNWARD : AMDGPUGenericInstruction {
+def G_FPTRUNC_ROUND : AMDGPUGenericInstruction {
   let OutOperandList = (outs type0:$vdst);
-  let InOperandList = (ins type1:$src0);
+  let InOperandList = (ins type1:$src0, untyped_imm_0:$round);
   let hasSideEffects = 0;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index c5251826b117cb8..4818f1b13caa6ae 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -8,7 +8,6 @@
 
 #include "SIMachineFunctionInfo.h"
 #include "AMDGPUSubtarget.h"
-#include "AMDGPUTargetMachine.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIRegisterInfo.h"
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index e7f448233ca347d..a590c6560942cf4 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -163,17 +163,19 @@ FunctionPass *llvm::createSIModeRegisterPass() { return new SIModeRegister(); }
 // double precision setting.
 Status SIModeRegister::getInstructionMode(MachineInstr &MI,
                                           const SIInstrInfo *TII) {
+  unsigned Opcode = MI.getOpcode();
   if (TII->usesFPDPRounding(MI) ||
-      MI.getOpcode() == AMDGPU::FPTRUNC_UPWARD_PSEUDO ||
-      MI.getOpcode() == AMDGPU::FPTRUNC_DOWNWARD_PSEUDO) {
-    switch (MI.getOpcode()) {
+      Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO) {
+    switch (Opcode) {
     case AMDGPU::V_INTERP_P1LL_F16:
     case AMDGPU::V_INTERP_P1LV_F16:
     case AMDGPU::V_INTERP_P2_F16:
       // f16 interpolation instructions need double precision round to zero
       return Status(FP_ROUND_MODE_DP(3),
                     FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_ZERO));
-    case AMDGPU::FPTRUNC_UPWARD_PSEUDO: {
+    case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO: {
+      unsigned Mode = MI.getOperand(2).getImm();
+      MI.removeOperand(2);
       // Replacing the pseudo by a real instruction in place
       if (TII->getSubtarget().hasTrue16BitInsts()) {
         MachineBasicBlock &MBB = *MI.getParent();
@@ -188,24 +190,7 @@ Status SIModeRegister::getInstructionMode(MachineInstr &MI,
       } else
         MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
       return Status(FP_ROUND_MODE_DP(3),
-                    FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_INF));
-    }
-    case AMDGPU::FPTRUNC_DOWNWARD_PSEUDO: {
-      // Replacing the pseudo by a real instruction in place
-      if (TII->getSubtarget().hasTrue16BitInsts()) {
-        MachineBasicBlock &MBB = *MI.getParent();
-        MachineInstrBuilder B(*MBB.getParent(), MI);
-        MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_t16_e64));
-        MachineOperand Src0 = MI.getOperand(1);
-        MI.removeOperand(1);
-        B.addImm(0); // src0_modifiers
-        B.add(Src0); // re-add src0 operand
-        B.addImm(0); // clamp
-        B.addImm(0); // omod
-      } else
-        MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
-      return Status(FP_ROUND_MODE_DP(3),
-                    FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEGINF));
+                    FP_ROUND_MODE_DP(Mode));
     }
     default:
       return DefaultStatus;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 0ca6266cc678b69..5b41a2cd731607b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -906,11 +906,7 @@ unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
 }
 
 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
-  unsigned BytesPerCU = 0;
-  if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
-    BytesPerCU = 32768;
-  if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
-    BytesPerCU = 65536;
+  unsigned BytesPerCU = getAddressableLocalMemorySize(STI);
 
   // "Per CU" really means "per whatever functional block the waves of a
   // workgroup must share". So the effective local memory size is doubled in
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
index 720d5a1853dbbd7..7b88ddb7b0e95d1 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
@@ -226,35 +226,35 @@ class PrintField {
   template <typename T, T AMDGPUMCKernelCodeT::*ptr,
             typename std::enable_if_t<!std::is_integral_v<T>, T> * = nullptr>
   static void printField(StringRef Name, const AMDGPUMCKernelCodeT &C,
-                         raw_ostream &OS, MCContext &Ctx) {
+                         raw_ostream &OS, MCContext &Ctx,
+                         AMDGPUMCKernelCodeT::PrintHelper Helper) {
     OS << Name << " = ";
     const MCExpr *Value = C.*ptr;
-    int64_t Val;
-    if (Value->evaluateAsAbsolute(Val))
-      OS << Val;
-    else
-      Value->print(OS, Ctx.getAsmInfo());
+    Helper(Value, OS, Ctx.getAsmInfo());
   }
 
   template <typename T, T AMDGPUMCKernelCodeT::*ptr,
             typename std::enable_if_t<std::is_integral_v<T>, T> * = nullptr>
   static void printField(StringRef Name, const AMDGPUMCKernelCodeT &C,
-                         raw_ostream &OS, MCContext &) {
+                         raw_ostream &OS, MCContext &,
+                         AMDGPUMCKernelCodeT::PrintHelper) {
     OS << Name << " = " << (int)(C.*ptr);
   }
 };
 
 template <typename T, T AMDGPUMCKernelCodeT::*ptr, int shift, int width = 1>
 static void printBitField(StringRef Name, const AMDGPUMCKernelCodeT &C,
-                          raw_ostream &OS, MCContext &) {
+                          raw_ostream &OS, MCContext &,
+                          AMDGPUMCKernelCodeT::PrintHelper) {
   const auto Mask = (static_cast<T>(1) << width) - 1;
   OS << Name << " = " << (int)((C.*ptr >> shift) & Mask);
 }
 
 using PrintFx = void (*)(StringRef, const AMDGPUMCKernelCodeT &, raw_ostream &,
-                         MCContext &);
+                         MCContext &, AMDGPUMCKernelCodeT::PrintHelper Helper);
 
-static ArrayRef<PrintFx> getPrinterTable() {
+static ArrayRef<PrintFx>
+getPrinterTable(AMDGPUMCKernelCodeT::PrintHelper Helper) {
   static const PrintFx Table[] = {
 #define COMPPGM1(name, aname, AccMacro)                                        \
   COMPPGM(name, aname, C_00B848_##AccMacro, S_00B848_##AccMacro, 0)
@@ -263,7 +263,7 @@ static ArrayRef<PrintFx> getPrinterTable() {
 #define PRINTFIELD(sname, aname, name) PrintField::printField<FLD_T(name)>
 #define PRINTCOMP(Complement, PGMType)                                         \
   [](StringRef Name, const AMDGPUMCKernelCodeT &C, raw_ostream &OS,            \
-     MCContext &Ctx) {                                                         \
+     MCContext &Ctx, AMDGPUMCKernelCodeT::PrintHelper Helper) {                \
     OS << Name << " = ";                                                       \
     auto [Shift, Mask] = getShiftMask(Complement);                             \
     const MCExpr *Value;                                                       \
@@ -274,11 +274,7 @@ static ArrayRef<PrintFx> getPrinterTable() {
       Value =                                                                  \
           maskShiftGet(C.compute_pgm_resource2_registers, Mask, Shift, Ctx);   \
     }                                                                          \
-    int64_t Val;                                                               \
-    if (Value->evaluateAsAbsolute(Val))                                        \
-      OS << Val;                                                               \
-    else                                                                       \
-      Value->print(OS, Ctx.getAsmInfo());                                      \
+    Helper(Value, OS, Ctx.getAsmInfo());                                       \
   }
 #define RECORD(name, altName, print, parse) print
 #include "Utils/AMDKernelCodeTInfo.h"
@@ -379,10 +375,11 @@ static ArrayRef<ParseFx> getParserTable() {
 }
 
 static void printAmdKernelCodeField(const AMDGPUMCKernelCodeT &C, int FldIndex,
-                                    raw_ostream &OS, MCContext &Ctx) {
-  auto Printer = getPrinterTable()[FldIndex];
+                                    raw_ostream &OS, MCContext &Ctx,
+                                    AMDGPUMCKernelCodeT::PrintHelper Helper) {
+  auto Printer = getPrinterTable(Helper)[FldIndex];
   if (Printer)
-    Printer(get_amd_kernel_code_t_FldNames()[FldIndex + 1], C, OS, Ctx);
+    Printer(get_amd_kernel_code_t_FldNames()[FldIndex + 1], C, OS, Ctx, Helper);
 }
 
 void AMDGPUMCKernelCodeT::initDefault(const MCSubtargetInfo *STI,
@@ -459,20 +456,17 @@ bool AMDGPUMCKernelCodeT::ParseKernelCodeT(StringRef ID, MCAsmParser &MCParser,
   return Parser ? Parser(*this, MCParser, Err) : false;
 }
 
-void AMDGPUMCKernelCodeT::EmitKernelCodeT(raw_ostream &OS, MCContext &Ctx) {
+void AMDGPUMCKernelCodeT::EmitKernelCodeT(raw_ostream &OS, MCContext &Ctx,
+                                          PrintHelper Helper) {
   const int Size = hasMCExprVersionTable().size();
   for (int i = 0; i < Size; ++i) {
     OS << "\t\t";
     if (hasMCExprVersionTable()[i]) {
       OS << get_amd_kernel_code_t_FldNames()[i + 1] << " = ";
-      int64_t Val;
       const MCExpr *Value = getMCExprForIndex(i);
-      if (Value->evaluateAsAbsolute(Val))
-        OS << Val;
-      else
-        Value->print(OS, Ctx.getAsmInfo());
+      Helper(Value, OS, Ctx.getAsmInfo());
     } else {
-      printAmdKernelCodeField(*this, i, OS, Ctx);
+      printAmdKernelCodeField(*this, i, OS, Ctx, Helper);
     }
     OS << '\n';
   }
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
index 6aeb98f1ce147b8..39acd5c43053bf5 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
@@ -27,6 +27,7 @@ class MCExpr;
 class MCStreamer;
 class MCSubtargetInfo;
 class raw_ostream;
+class MCAsmInfo;
 namespace AMDGPU {
 
 struct AMDGPUMCKernelCodeT {
@@ -79,8 +80,10 @@ struct AMDGPUMCKernelCodeT {
 
   const MCExpr *&getMCExprForIndex(int Index);
 
+  using PrintHelper =
+      function_ref<void(const MCExpr *, raw_ostream &, const MCAsmInfo *)>;
   bool ParseKernelCodeT(StringRef ID, MCAsmParser &MCParser, raw_ostream &Err);
-  void EmitKernelCodeT(raw_ostream &OS, MCContext &Ctx);
+  void EmitKernelCodeT(raw_ostream &OS, MCContext &Ctx, PrintHelper Helper);
   void EmitKernelCodeT(MCStreamer &OS, MCContext &Ctx);
 };
 
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 3b9195b4e3d85fe..1199052ca97e9c7 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -6074,8 +6074,8 @@ ARMBaseInstrInfo::getOutliningCandidateInfo(
     // check if the range contains a call.  These require a save + restore of
     // the link register.
     outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
-    if (std::any_of(FirstCand.begin(), std::prev(FirstCand.end()),
-                    [](const MachineInstr &MI) { return MI.isCall(); }))
+    if (any_of(drop_end(FirstCand),
+               [](const MachineInstr &MI) { return MI.isCall(); }))
       NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
 
     // Handle the last instruction separately.  If it is tail call, then the
diff --git a/llvm/lib/Target/ARM/ARMCallLowering.cpp b/llvm/lib/Target/ARM/ARMCallLowering.cpp
index 9cc162d041f48b8..883808ae981f5e4 100644
--- a/llvm/lib/Target/ARM/ARMCallLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMCallLowering.cpp
@@ -50,6 +50,13 @@
 
 using namespace llvm;
 
+// Whether Big-endian GISel is enabled, defaults to off, can be enabled for
+// testing.
+static cl::opt<bool>
+    EnableGISelBigEndian("enable-arm-gisel-bigendian", cl::Hidden,
+                         cl::init(false),
+                         cl::desc("Enable Global-ISel Big Endian Lowering"));
+
 ARMCallLowering::ARMCallLowering(const ARMTargetLowering &TLI)
     : CallLowering(&TLI) {}
 
@@ -539,3 +546,5 @@ bool ARMCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &
 
   return true;
 }
+
+bool ARMCallLowering::enableBigEndian() const { return EnableGISelBigEndian; }
\ No newline at end of file
diff --git a/llvm/lib/Target/ARM/ARMCallLowering.h b/llvm/lib/Target/ARM/ARMCallLowering.h
index 38095617fb4f354..32c95a044d7b7ea 100644
--- a/llvm/lib/Target/ARM/ARMCallLowering.h
+++ b/llvm/lib/Target/ARM/ARMCallLowering.h
@@ -42,6 +42,8 @@ class ARMCallLowering : public CallLowering {
   bool lowerCall(MachineIRBuilder &MIRBuilder,
                  CallLoweringInfo &Info) const override;
 
+  bool enableBigEndian() const override;
+
 private:
   bool lowerReturnVal(MachineIRBuilder &MIRBuilder, const Value *Val,
                       ArrayRef<Register> VRegs,
diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index 7ffc64378ef4baf..f1b579ce7d2d9c9 100644
--- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -712,7 +712,8 @@ bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
         Base = CurDAG->getTargetFrameIndex(
             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                         /*isTarget=*/true);
       return true;
     }
   }
@@ -881,7 +882,8 @@ bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
     if (AddSub == ARM_AM::sub) Val *= -1;
     Offset = CurDAG->getRegister(0, MVT::i32);
-    Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
+    Opc =
+        CurDAG->getSignedConstant(Val, SDLoc(Op), MVT::i32, /*isTarget*/ true);
     return true;
   }
 
@@ -1185,7 +1187,8 @@ ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
   int RHSC;
   if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
     Base = N.getOperand(0);
-    OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+    OffImm =
+        CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32, /*isTarget=*/true);
     return true;
   }
 
@@ -1247,7 +1250,8 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
         if (MFI.getObjectAlign(FI) >= Align(4)) {
           Base = CurDAG->getTargetFrameIndex(
               FI, TLI->getPointerTy(CurDAG->getDataLayout()));
-          OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+          OffImm = CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                             /*isTarget=*/true);
           return true;
         }
       }
@@ -1267,8 +1271,8 @@ bool ARMDAGToDAGISel::SelectTAddrModeImm7(SDValue N, SDValue &Base,
       Base = N.getOperand(0);
       if (N.getOpcode() == ISD::SUB)
         RHSC = -RHSC;
-      OffImm =
-          CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC * (1 << Shift), SDLoc(N),
+                                         MVT::i32, /*isTarget=*/true);
       return true;
     }
   }
@@ -1330,7 +1334,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
         Base = CurDAG->getTargetFrameIndex(
             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                         /*isTarget=*/true);
       return true;
     }
   }
@@ -1357,7 +1362,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, SDValue &Base,
       if (N.getOpcode() == ISD::SUB)
         RHSC = -RHSC;
       OffImm =
-          CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
+          CurDAG->getSignedConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32,
+                                    /*isTarget=*/true);
       return true;
     }
   }
@@ -1387,7 +1393,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
         Base = CurDAG->getTargetFrameIndex(
             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                         /*isTarget=*/true);
       return true;
     }
   }
@@ -1404,8 +1411,10 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
   int RHSC;
   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
     OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
-      ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
-      : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
+                 ? CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                             /*isTarget=*/true)
+                 : CurDAG->getSignedConstant(-RHSC, SDLoc(N), MVT::i32,
+                                             /*isTarget=*/true);
     return true;
   }
 
@@ -1428,8 +1437,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm7(SDValue N, SDValue &Base,
 
       if (N.getOpcode() == ISD::SUB)
         RHSC = -RHSC;
-      OffImm =
-          CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC * (1 << Shift), SDLoc(N),
+                                         MVT::i32, /*isTarget=*/true);
       return true;
     }
   }
@@ -1471,11 +1480,11 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N,
   int RHSC;
   // 7 bit constant, shifted by Shift.
   if (isScaledConstantInRange(N, 1 << Shift, 0, 0x80, RHSC)) {
-    OffImm =
-        ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
-            ? CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32)
-            : CurDAG->getTargetConstant(-RHSC * (1 << Shift), SDLoc(N),
-                                        MVT::i32);
+    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
+                 ? CurDAG->getSignedConstant(RHSC * (1 << Shift), SDLoc(N),
+                                             MVT::i32, /*isTarget=*/true)
+                 : CurDAG->getSignedConstant(-RHSC * (1 << Shift), SDLoc(N),
+                                             MVT::i32, /*isTarget=*/true);
     return true;
   }
   return false;
@@ -1485,7 +1494,8 @@ template <int Min, int Max>
 bool ARMDAGToDAGISel::SelectImmediateInRange(SDValue N, SDValue &OffImm) {
   int Val;
   if (isScaledConstantInRange(N, 1, Min, Max, Val)) {
-    OffImm = CurDAG->getTargetConstant(Val, SDLoc(N), MVT::i32);
+    OffImm =
+        CurDAG->getSignedConstant(Val, SDLoc(N), MVT::i32, /*isTarget=*/true);
     return true;
   }
   return false;
@@ -3861,8 +3871,7 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
               ConstantMaterializationCost(~Imm, Subtarget)) {
         // The current immediate costs more to materialize than a negated
         // immediate, so negate the immediate and use a BIC.
-        SDValue NewImm =
-          CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
+        SDValue NewImm = CurDAG->getConstant(~Imm, dl, MVT::i32);
         // If the new constant didn't exist before, reposition it in the topological
         // ordering so it is just before N. Otherwise, don't touch its location.
         if (NewImm->getNodeId() == -1)
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 1fab30a0b855088..1e8bb8a495e68bb 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -2841,7 +2841,8 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   Ops.push_back(Callee);
 
   if (isTailCall) {
-    Ops.push_back(DAG.getTargetConstant(SPDiff, dl, MVT::i32));
+    Ops.push_back(
+        DAG.getSignedConstant(SPDiff, dl, MVT::i32, /*isTarget=*/true));
   }
 
   // Add argument registers to the end of the list so that they are known live
@@ -2892,7 +2893,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // we need to undo that after it returns to restore the status-quo.
   bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
   uint64_t CalleePopBytes =
-      canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1ULL;
+      canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1U;
 
   Chain = DAG.getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
   if (!Ins.empty())
@@ -8551,7 +8552,7 @@ static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
 
   SmallVector<SDValue, 8> VTBLMask;
   for (int I : ShuffleMask)
-    VTBLMask.push_back(DAG.getConstant(I, DL, MVT::i32));
+    VTBLMask.push_back(DAG.getSignedConstant(I, DL, MVT::i32));
 
   if (V2.getNode()->isUndef())
     return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
@@ -20671,7 +20672,8 @@ void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
         }
         return;
     }
-    Result = DAG.getTargetConstant(CVal, SDLoc(Op), Op.getValueType());
+    Result = DAG.getSignedConstant(CVal, SDLoc(Op), Op.getValueType(),
+                                   /*isTarget=*/true);
     break;
   }
 
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index c6bcad8e2a82c52..26f7d70b43b2623 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -371,12 +371,14 @@ def ARMVCCElse : PatLeaf<(i32 2)>;
 
 // imm_neg_XFORM - Return the negation of an i32 immediate value.
 def imm_neg_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(-(int)N->getZExtValue(), SDLoc(N), MVT::i32);
+  return CurDAG->getSignedConstant(-(int)N->getZExtValue(), SDLoc(N), MVT::i32,
+                                   /*isTarget=*/true);
 }]>;
 
 // imm_not_XFORM - Return the complement of a i32 immediate value.
 def imm_not_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(~(int)N->getZExtValue(), SDLoc(N), MVT::i32);
+  return CurDAG->getSignedConstant(~(int)N->getZExtValue(), SDLoc(N), MVT::i32,
+                                   /*isTarget=*/true);
 }]>;
 def gi_imm_not_XFORM : GICustomOperandRenderer<"renderInvertedImm">,
   GISDNodeXFormEquiv<imm_not_XFORM>;
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index e54314cc7d00afa..c2951bf6dbf78f8 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -13090,9 +13090,9 @@ bool ARMAsmParser::isMnemonicVPTPredicable(StringRef Mnemonic,
       "vshrn",      "vsli",     "vsri",      "vstrb",      "vstrd",
       "vstrw",      "vsub"};
 
-  return std::any_of(
-      std::begin(predicable_prefixes), std::end(predicable_prefixes),
-      [&Mnemonic](const char *prefix) { return Mnemonic.starts_with(prefix); });
+  return any_of(predicable_prefixes, [&Mnemonic](const char *prefix) {
+    return Mnemonic.starts_with(prefix);
+  });
 }
 
 std::unique_ptr<ARMOperand> ARMAsmParser::defaultCondCodeOp() {
diff --git a/llvm/lib/Target/DirectX/CMakeLists.txt b/llvm/lib/Target/DirectX/CMakeLists.txt
index 4c70b3f9230edba..f7ae09957996b50 100644
--- a/llvm/lib/Target/DirectX/CMakeLists.txt
+++ b/llvm/lib/Target/DirectX/CMakeLists.txt
@@ -19,6 +19,7 @@ add_llvm_target(DirectXCodeGen
   DirectXSubtarget.cpp
   DirectXTargetMachine.cpp
   DXContainerGlobals.cpp
+  DXILFinalizeLinkage.cpp
   DXILIntrinsicExpansion.cpp
   DXILMetadata.cpp
   DXILOpBuilder.cpp
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 67015cff78a79a0..34c7f84b1ca5b2e 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -24,19 +24,24 @@ foreach i = 0...8 in {
   def DXIL1_ #i : Version<1, i>;
 }
 
-// Overload type alias of llvm_any_ty
-defvar overloadTy = llvm_any_ty;
-
-// Type aliases for DXIL Op types to LLVM Types.
-// TODO: Define DXIL Types independent of LLVM types
-defvar i1Ty = llvm_i1_ty;
-defvar i8Ty = llvm_i8_ty;
-defvar i16Ty = llvm_i16_ty;
-defvar i32Ty = llvm_i32_ty;
-defvar i64Ty = llvm_i64_ty;
-defvar halfTy = llvm_half_ty;
-defvar floatTy = llvm_float_ty;
-defvar doubleTy = llvm_double_ty;
+class DXILOpParamType {
+  int isOverload = 0;
+}
+
+let isOverload = 1 in {
+  def OverloadTy : DXILOpParamType;
+}
+def VoidTy : DXILOpParamType;
+def Int1Ty : DXILOpParamType;
+def Int8Ty : DXILOpParamType;
+def Int16Ty : DXILOpParamType;
+def Int32Ty : DXILOpParamType;
+def Int64Ty : DXILOpParamType;
+def HalfTy : DXILOpParamType;
+def FloatTy : DXILOpParamType;
+def DoubleTy : DXILOpParamType;
+def ResRetTy : DXILOpParamType;
+def HandleTy : DXILOpParamType;
 
 class DXILOpClass;
 
@@ -268,9 +273,9 @@ def IsWave : DXILAttribute;
 def NeedsUniformInputs : DXILAttribute;
 def IsBarrier : DXILAttribute;
 
-class Overloads<Version ver, list<LLVMType> ols> {
+class Overloads<Version ver, list<DXILOpParamType> ols> {
   Version dxil_version = ver;
-  list<LLVMType> overload_types = ols;
+  list<DXILOpParamType> overload_types = ols;
 }
 
 class Stages<Version ver, list<DXILShaderStage> st> {
@@ -298,10 +303,10 @@ class DXILOp<int opcode, DXILOpClass opclass> {
   Intrinsic LLVMIntrinsic = ?;
 
   // Result type of the op
-  LLVMType result;
+  DXILOpParamType result;
 
   // List of argument types of the op. Default to 0 arguments.
-  list<LLVMType> arguments = [];
+  list<DXILOpParamType> arguments = [];
 
   // List of valid overload types predicated by DXIL version
   list<Overloads> overloads = [];
@@ -318,9 +323,9 @@ class DXILOp<int opcode, DXILOpClass opclass> {
 def Abs :  DXILOp<6, unary> {
   let Doc = "Returns the absolute value of the input.";
   let LLVMIntrinsic = int_fabs;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy, doubleTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -328,9 +333,9 @@ def Abs :  DXILOp<6, unary> {
 def IsInf :  DXILOp<9, isSpecialFloat> {
   let Doc = "Determines if the specified value is infinite.";
   let LLVMIntrinsic = int_dx_isinf;
-  let arguments = [overloadTy];
-  let result = i1Ty;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = Int1Ty;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -338,9 +343,9 @@ def IsInf :  DXILOp<9, isSpecialFloat> {
 def Cos :  DXILOp<12, unary> {
   let Doc = "Returns cosine(theta) for theta in radians.";
   let LLVMIntrinsic = int_cos;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -348,9 +353,9 @@ def Cos :  DXILOp<12, unary> {
 def Sin :  DXILOp<13, unary> {
   let Doc = "Returns sine(theta) for theta in radians.";
   let LLVMIntrinsic = int_sin;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -358,9 +363,9 @@ def Sin :  DXILOp<13, unary> {
 def Tan :  DXILOp<14, unary> {
   let Doc = "Returns tangent(theta) for theta in radians.";
   let LLVMIntrinsic = int_tan;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -368,9 +373,9 @@ def Tan :  DXILOp<14, unary> {
 def ACos :  DXILOp<15, unary> {
   let Doc = "Returns the arccosine of the specified value.";
   let LLVMIntrinsic = int_acos;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -378,9 +383,9 @@ def ACos :  DXILOp<15, unary> {
 def ASin :  DXILOp<16, unary> {
   let Doc = "Returns the arcsine of the specified value.";
   let LLVMIntrinsic = int_asin;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -388,9 +393,9 @@ def ASin :  DXILOp<16, unary> {
 def ATan :  DXILOp<17, unary> {
   let Doc = "Returns the arctangent of the specified value.";
   let LLVMIntrinsic = int_atan;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -398,9 +403,9 @@ def ATan :  DXILOp<17, unary> {
 def HCos :  DXILOp<18, unary> {
   let Doc = "Returns the hyperbolic cosine of the specified value.";
   let LLVMIntrinsic = int_cosh;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -408,9 +413,9 @@ def HCos :  DXILOp<18, unary> {
 def HSin :  DXILOp<19, unary> {
   let Doc = "Returns the hyperbolic sine of the specified value.";
   let LLVMIntrinsic = int_sinh;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -418,9 +423,9 @@ def HSin :  DXILOp<19, unary> {
 def HTan :  DXILOp<20, unary> {
   let Doc = "Returns the hyperbolic tan of the specified value.";
   let LLVMIntrinsic = int_tanh;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -429,9 +434,9 @@ def Exp2 :  DXILOp<21, unary> {
   let Doc = "Returns the base 2 exponential, or 2**x, of the specified value. "
             "exp2(x) = 2**x.";
   let LLVMIntrinsic = int_exp2;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -440,9 +445,9 @@ def Frac :  DXILOp<22, unary> {
   let Doc = "Returns a fraction from 0 to 1 that represents the decimal part "
             "of the input.";
   let LLVMIntrinsic = int_dx_frac;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -450,9 +455,9 @@ def Frac :  DXILOp<22, unary> {
 def Log2 :  DXILOp<23, unary> {
   let Doc = "Returns the base-2 logarithm of the specified value.";
   let LLVMIntrinsic = int_log2;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -461,9 +466,9 @@ def Sqrt :  DXILOp<24, unary> {
   let Doc = "Returns the square root of the specified floating-point value, "
             "per component.";
   let LLVMIntrinsic = int_sqrt;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -472,9 +477,9 @@ def RSqrt :  DXILOp<25, unary> {
   let Doc = "Returns the reciprocal of the square root of the specified value. "
             "rsqrt(x) = 1 / sqrt(x).";
   let LLVMIntrinsic = int_dx_rsqrt;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -483,9 +488,9 @@ def Round :  DXILOp<26, unary> {
   let Doc = "Returns the input rounded to the nearest integer within a "
             "floating-point type.";
   let LLVMIntrinsic = int_roundeven;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -494,9 +499,9 @@ def Floor :  DXILOp<27, unary> {
   let Doc =
       "Returns the largest integer that is less than or equal to the input.";
   let LLVMIntrinsic = int_floor;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -505,9 +510,9 @@ def Ceil :  DXILOp<28, unary> {
   let Doc = "Returns the smallest integer that is greater than or equal to the "
             "input.";
   let LLVMIntrinsic = int_ceil;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -515,9 +520,9 @@ def Ceil :  DXILOp<28, unary> {
 def Trunc :  DXILOp<29, unary> {
   let Doc = "Returns the specified value truncated to the integer component.";
   let LLVMIntrinsic = int_trunc;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -525,10 +530,10 @@ def Trunc :  DXILOp<29, unary> {
 def Rbits :  DXILOp<30, unary> {
   let Doc = "Returns the specified value with its bits reversed.";
   let LLVMIntrinsic = int_bitreverse;
-  let arguments = [overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -536,10 +541,10 @@ def Rbits :  DXILOp<30, unary> {
 def FMax :  DXILOp<35, binary> {
   let Doc = "Float maximum. FMax(a,b) = a > b ? a : b";
   let LLVMIntrinsic = int_maxnum;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [halfTy, floatTy, doubleTy]>];
+      [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -547,10 +552,10 @@ def FMax :  DXILOp<35, binary> {
 def FMin :  DXILOp<36, binary> {
   let Doc = "Float minimum. FMin(a,b) = a < b ? a : b";
   let LLVMIntrinsic = int_minnum;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [halfTy, floatTy, doubleTy]>];
+      [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -558,10 +563,10 @@ def FMin :  DXILOp<36, binary> {
 def SMax :  DXILOp<37, binary> {
   let Doc = "Signed integer maximum. SMax(a,b) = a > b ? a : b";
   let LLVMIntrinsic = int_smax;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -569,10 +574,10 @@ def SMax :  DXILOp<37, binary> {
 def SMin :  DXILOp<38, binary> {
   let Doc = "Signed integer minimum. SMin(a,b) = a < b ? a : b";
   let LLVMIntrinsic = int_smin;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -580,10 +585,10 @@ def SMin :  DXILOp<38, binary> {
 def UMax :  DXILOp<39, binary> {
   let Doc = "Unsigned integer maximum. UMax(a,b) = a > b ? a : b";
   let LLVMIntrinsic = int_umax;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -591,10 +596,10 @@ def UMax :  DXILOp<39, binary> {
 def UMin :  DXILOp<40, binary> {
   let Doc = "Unsigned integer minimum. UMin(a,b) = a < b ? a : b";
   let LLVMIntrinsic = int_umin;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -603,10 +608,10 @@ def FMad :  DXILOp<46, tertiary> {
   let Doc = "Floating point arithmetic multiply/add operation. fmad(m,a,b) = m "
             "* a + b.";
   let LLVMIntrinsic = int_fmuladd;
-  let arguments = [overloadTy, overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [halfTy, floatTy, doubleTy]>];
+      [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -615,10 +620,10 @@ def IMad :  DXILOp<48, tertiary> {
   let Doc = "Signed integer arithmetic multiply/add operation. imad(m,a,b) = m "
             "* a + b.";
   let LLVMIntrinsic = int_dx_imad;
-  let arguments = [overloadTy, overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -627,10 +632,10 @@ def UMad :  DXILOp<49, tertiary> {
   let Doc = "Unsigned integer arithmetic multiply/add operation. umad(m,a, = m "
             "* a + b.";
   let LLVMIntrinsic = int_dx_umad;
-  let arguments = [overloadTy, overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -639,9 +644,9 @@ def Dot2 :  DXILOp<54, dot2> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is between 0 and 1";
   let LLVMIntrinsic = int_dx_dot2;
-  let arguments = !listsplat(overloadTy, 4);
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = !listsplat(OverloadTy, 4);
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -650,9 +655,9 @@ def Dot3 :  DXILOp<55, dot3> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is between 0 and 2";
   let LLVMIntrinsic = int_dx_dot3;
-  let arguments = !listsplat(overloadTy, 6);
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = !listsplat(OverloadTy, 6);
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -661,9 +666,9 @@ def Dot4 :  DXILOp<56, dot4> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is between 0 and 3";
   let LLVMIntrinsic = int_dx_dot4;
-  let arguments = !listsplat(overloadTy, 8);
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = !listsplat(OverloadTy, 8);
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -671,8 +676,9 @@ def Dot4 :  DXILOp<56, dot4> {
 def ThreadId :  DXILOp<93, threadId> {
   let Doc = "Reads the thread ID";
   let LLVMIntrinsic = int_dx_thread_id;
-  let arguments = [i32Ty];
-  let result = i32Ty;
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -680,8 +686,9 @@ def ThreadId :  DXILOp<93, threadId> {
 def GroupId :  DXILOp<94, groupId> {
   let Doc = "Reads the group ID (SV_GroupID)";
   let LLVMIntrinsic = int_dx_group_id;
-  let arguments = [i32Ty];
-  let result = i32Ty;
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -689,8 +696,9 @@ def GroupId :  DXILOp<94, groupId> {
 def ThreadIdInGroup :  DXILOp<95, threadIdInGroup> {
   let Doc = "Reads the thread ID within the group  (SV_GroupThreadID)";
   let LLVMIntrinsic = int_dx_thread_id_in_group;
-  let arguments = [i32Ty];
-  let result = i32Ty;
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -699,7 +707,8 @@ def FlattenedThreadIdInGroup :  DXILOp<96, flattenedThreadIdInGroup> {
   let Doc = "Provides a flattened index for a given thread within a given "
             "group (SV_GroupIndex)";
   let LLVMIntrinsic = int_dx_flattened_thread_id_in_group;
-  let result = i32Ty;
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
diff --git a/llvm/lib/Target/DirectX/DXILConstants.h b/llvm/lib/Target/DirectX/DXILConstants.h
index 0c9c1ac38fdbceb..022cd57795a0633 100644
--- a/llvm/lib/Target/DirectX/DXILConstants.h
+++ b/llvm/lib/Target/DirectX/DXILConstants.h
@@ -25,6 +25,11 @@ enum class OpCodeClass : unsigned {
 #include "DXILOperation.inc"
 };
 
+enum class OpParamType : unsigned {
+#define DXIL_OP_PARAM_TYPE(Name) Name,
+#include "DXILOperation.inc"
+};
+
 } // namespace dxil
 } // namespace llvm
 
diff --git a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
new file mode 100644
index 000000000000000..c02eb768cdf49bb
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
@@ -0,0 +1,60 @@
+//===- DXILFinalizeLinkage.cpp - Finalize linkage of functions ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DXILFinalizeLinkage.h"
+#include "DirectX.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+
+#define DEBUG_TYPE "dxil-finalize-linkage"
+
+using namespace llvm;
+
+static bool finalizeLinkage(Module &M) {
+  SmallPtrSet<Function *, 8> EntriesAndExports;
+
+  // Find all entry points and export functions
+  for (Function &EF : M.functions()) {
+    if (!EF.hasFnAttribute("hlsl.shader") && !EF.hasFnAttribute("hlsl.export"))
+      continue;
+    EntriesAndExports.insert(&EF);
+  }
+
+  for (Function &F : M.functions()) {
+    if (F.getLinkage() == GlobalValue::ExternalLinkage &&
+        !EntriesAndExports.contains(&F)) {
+      F.setLinkage(GlobalValue::InternalLinkage);
+    }
+  }
+
+  return false;
+}
+
+PreservedAnalyses DXILFinalizeLinkage::run(Module &M,
+                                           ModuleAnalysisManager &AM) {
+  if (finalizeLinkage(M))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}
+
+bool DXILFinalizeLinkageLegacy::runOnModule(Module &M) {
+  return finalizeLinkage(M);
+}
+
+char DXILFinalizeLinkageLegacy::ID = 0;
+
+INITIALIZE_PASS_BEGIN(DXILFinalizeLinkageLegacy, DEBUG_TYPE,
+                      "DXIL Finalize Linkage", false, false)
+INITIALIZE_PASS_END(DXILFinalizeLinkageLegacy, DEBUG_TYPE,
+                    "DXIL Finalize Linkage", false, false)
+
+ModulePass *llvm::createDXILFinalizeLinkageLegacyPass() {
+  return new DXILFinalizeLinkageLegacy();
+}
diff --git a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.h b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.h
new file mode 100644
index 000000000000000..aab1bc3f7a28e20
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.h
@@ -0,0 +1,39 @@
+//===- DXILFinalizeLinkage.h - Finalize linkage of functions --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// DXILFinalizeLinkage pass updates the linkage of functions to make sure only
+/// shader entry points and exported functions are visible from the module (have
+/// program linkage). All other functions will be updated to have internal
+/// linkage.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_DIRECTX_DXILFINALIZELINKAGE_H
+#define LLVM_TARGET_DIRECTX_DXILFINALIZELINKAGE_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/Pass.h"
+
+namespace llvm {
+
+class DXILFinalizeLinkage : public PassInfoMixin<DXILFinalizeLinkage> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  static bool isRequired() { return true; }
+};
+
+class DXILFinalizeLinkageLegacy : public ModulePass {
+public:
+  DXILFinalizeLinkageLegacy() : ModulePass(ID) {}
+  bool runOnModule(Module &M) override;
+
+  static char ID; // Pass identification.
+};
+} // namespace llvm
+
+#endif // LLVM_TARGET_DIRECTX_DXILFINALIZELINKAGE_H
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index e63633b8a1e1abc..2c481d15be5bde3 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -51,7 +51,7 @@ static bool isIntrinsicExpansion(Function &F) {
   return false;
 }
 
-static bool expandAbs(CallInst *Orig) {
+static Value *expandAbs(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
@@ -64,14 +64,11 @@ static bool expandAbs(CallInst *Orig) {
                              ConstantInt::get(EltTy, 0))
                        : ConstantInt::get(EltTy, 0);
   auto *V = Builder.CreateSub(Zero, X);
-  auto *MaxCall =
-      Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr, "dx.max");
-  Orig->replaceAllUsesWith(MaxCall);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr,
+                                 "dx.max");
 }
 
-static bool expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) {
+static Value *expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) {
   assert(DotIntrinsic == Intrinsic::dx_sdot ||
          DotIntrinsic == Intrinsic::dx_udot);
   Intrinsic::ID MadIntrinsic = DotIntrinsic == Intrinsic::dx_sdot
@@ -97,12 +94,10 @@ static bool expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) {
                                      ArrayRef<Value *>{Elt0, Elt1, Result},
                                      nullptr, "dx.mad");
   }
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Result;
 }
 
-static bool expandExpIntrinsic(CallInst *Orig) {
+static Value *expandExpIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
@@ -119,23 +114,21 @@ static bool expandExpIntrinsic(CallInst *Orig) {
       Builder.CreateIntrinsic(Ty, Intrinsic::exp2, {NewX}, nullptr, "dx.exp2");
   Exp2Call->setTailCall(Orig->isTailCall());
   Exp2Call->setAttributes(Orig->getAttributes());
-  Orig->replaceAllUsesWith(Exp2Call);
-  Orig->eraseFromParent();
-  return true;
+  return Exp2Call;
 }
 
-static bool expandAnyIntrinsic(CallInst *Orig) {
+static Value *expandAnyIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
   Type *Ty = X->getType();
   Type *EltTy = Ty->getScalarType();
 
+  Value *Result = nullptr;
   if (!Ty->isVectorTy()) {
-    Value *Cond = EltTy->isFloatingPointTy()
-                      ? Builder.CreateFCmpUNE(X, ConstantFP::get(EltTy, 0))
-                      : Builder.CreateICmpNE(X, ConstantInt::get(EltTy, 0));
-    Orig->replaceAllUsesWith(Cond);
+    Result = EltTy->isFloatingPointTy()
+                 ? Builder.CreateFCmpUNE(X, ConstantFP::get(EltTy, 0))
+                 : Builder.CreateICmpNE(X, ConstantInt::get(EltTy, 0));
   } else {
     auto *XVec = dyn_cast<FixedVectorType>(Ty);
     Value *Cond =
@@ -148,18 +141,16 @@ static bool expandAnyIntrinsic(CallInst *Orig) {
                   X, ConstantVector::getSplat(
                          ElementCount::getFixed(XVec->getNumElements()),
                          ConstantInt::get(EltTy, 0)));
-    Value *Result = Builder.CreateExtractElement(Cond, (uint64_t)0);
+    Result = Builder.CreateExtractElement(Cond, (uint64_t)0);
     for (unsigned I = 1; I < XVec->getNumElements(); I++) {
       Value *Elt = Builder.CreateExtractElement(Cond, I);
       Result = Builder.CreateOr(Result, Elt);
     }
-    Orig->replaceAllUsesWith(Result);
   }
-  Orig->eraseFromParent();
-  return true;
+  return Result;
 }
 
-static bool expandLengthIntrinsic(CallInst *Orig) {
+static Value *expandLengthIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
@@ -182,15 +173,11 @@ static bool expandLengthIntrinsic(CallInst *Orig) {
     Value *Mul = Builder.CreateFMul(Elt, Elt);
     Sum = Builder.CreateFAdd(Sum, Mul);
   }
-  Value *Result = Builder.CreateIntrinsic(
-      EltTy, Intrinsic::sqrt, ArrayRef<Value *>{Sum}, nullptr, "elt.sqrt");
-
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateIntrinsic(EltTy, Intrinsic::sqrt, ArrayRef<Value *>{Sum},
+                                 nullptr, "elt.sqrt");
 }
 
-static bool expandLerpIntrinsic(CallInst *Orig) {
+static Value *expandLerpIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   Value *Y = Orig->getOperand(1);
   Value *S = Orig->getOperand(2);
@@ -198,14 +185,11 @@ static bool expandLerpIntrinsic(CallInst *Orig) {
   Builder.SetInsertPoint(Orig);
   auto *V = Builder.CreateFSub(Y, X);
   V = Builder.CreateFMul(S, V);
-  auto *Result = Builder.CreateFAdd(X, V, "dx.lerp");
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateFAdd(X, V, "dx.lerp");
 }
 
-static bool expandLogIntrinsic(CallInst *Orig,
-                               float LogConstVal = numbers::ln2f) {
+static Value *expandLogIntrinsic(CallInst *Orig,
+                                 float LogConstVal = numbers::ln2f) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
@@ -221,16 +205,13 @@ static bool expandLogIntrinsic(CallInst *Orig,
       Builder.CreateIntrinsic(Ty, Intrinsic::log2, {X}, nullptr, "elt.log2");
   Log2Call->setTailCall(Orig->isTailCall());
   Log2Call->setAttributes(Orig->getAttributes());
-  auto *Result = Builder.CreateFMul(Ln2Const, Log2Call);
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateFMul(Ln2Const, Log2Call);
 }
-static bool expandLog10Intrinsic(CallInst *Orig) {
+static Value *expandLog10Intrinsic(CallInst *Orig) {
   return expandLogIntrinsic(Orig, numbers::ln2f / numbers::ln10f);
 }
 
-static bool expandNormalizeIntrinsic(CallInst *Orig) {
+static Value *expandNormalizeIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   Type *Ty = Orig->getType();
   Type *EltTy = Ty->getScalarType();
@@ -245,11 +226,7 @@ static bool expandNormalizeIntrinsic(CallInst *Orig) {
         report_fatal_error(Twine("Invalid input scalar: length is zero"),
                            /* gen_crash_diag=*/false);
     }
-    Value *Result = Builder.CreateFDiv(X, X);
-
-    Orig->replaceAllUsesWith(Result);
-    Orig->eraseFromParent();
-    return true;
+    return Builder.CreateFDiv(X, X);
   }
 
   unsigned XVecSize = XVec->getNumElements();
@@ -291,14 +268,10 @@ static bool expandNormalizeIntrinsic(CallInst *Orig) {
                                                 nullptr, "dx.rsqrt");
 
   Value *MultiplicandVec = Builder.CreateVectorSplat(XVecSize, Multiplicand);
-  Value *Result = Builder.CreateFMul(X, MultiplicandVec);
-
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateFMul(X, MultiplicandVec);
 }
 
-static bool expandPowIntrinsic(CallInst *Orig) {
+static Value *expandPowIntrinsic(CallInst *Orig) {
 
   Value *X = Orig->getOperand(0);
   Value *Y = Orig->getOperand(1);
@@ -313,9 +286,7 @@ static bool expandPowIntrinsic(CallInst *Orig) {
       Builder.CreateIntrinsic(Ty, Intrinsic::exp2, {Mul}, nullptr, "elt.exp2");
   Exp2Call->setTailCall(Orig->isTailCall());
   Exp2Call->setAttributes(Orig->getAttributes());
-  Orig->replaceAllUsesWith(Exp2Call);
-  Orig->eraseFromParent();
-  return true;
+  return Exp2Call;
 }
 
 static Intrinsic::ID getMaxForClamp(Type *ElemTy,
@@ -344,7 +315,8 @@ static Intrinsic::ID getMinForClamp(Type *ElemTy,
   return Intrinsic::minnum;
 }
 
-static bool expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic) {
+static Value *expandClampIntrinsic(CallInst *Orig,
+                                   Intrinsic::ID ClampIntrinsic) {
   Value *X = Orig->getOperand(0);
   Value *Min = Orig->getOperand(1);
   Value *Max = Orig->getOperand(2);
@@ -353,41 +325,54 @@ static bool expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic) {
   Builder.SetInsertPoint(Orig);
   auto *MaxCall = Builder.CreateIntrinsic(
       Ty, getMaxForClamp(Ty, ClampIntrinsic), {X, Min}, nullptr, "dx.max");
-  auto *MinCall =
-      Builder.CreateIntrinsic(Ty, getMinForClamp(Ty, ClampIntrinsic),
-                              {MaxCall, Max}, nullptr, "dx.min");
-
-  Orig->replaceAllUsesWith(MinCall);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateIntrinsic(Ty, getMinForClamp(Ty, ClampIntrinsic),
+                                 {MaxCall, Max}, nullptr, "dx.min");
 }
 
 static bool expandIntrinsic(Function &F, CallInst *Orig) {
+  Value *Result = nullptr;
   switch (F.getIntrinsicID()) {
   case Intrinsic::abs:
-    return expandAbs(Orig);
+    Result = expandAbs(Orig);
+    break;
   case Intrinsic::exp:
-    return expandExpIntrinsic(Orig);
+    Result = expandExpIntrinsic(Orig);
+    break;
   case Intrinsic::log:
-    return expandLogIntrinsic(Orig);
+    Result = expandLogIntrinsic(Orig);
+    break;
   case Intrinsic::log10:
-    return expandLog10Intrinsic(Orig);
+    Result = expandLog10Intrinsic(Orig);
+    break;
   case Intrinsic::pow:
-    return expandPowIntrinsic(Orig);
+    Result = expandPowIntrinsic(Orig);
+    break;
   case Intrinsic::dx_any:
-    return expandAnyIntrinsic(Orig);
+    Result = expandAnyIntrinsic(Orig);
+    break;
   case Intrinsic::dx_uclamp:
   case Intrinsic::dx_clamp:
-    return expandClampIntrinsic(Orig, F.getIntrinsicID());
+    Result = expandClampIntrinsic(Orig, F.getIntrinsicID());
+    break;
   case Intrinsic::dx_lerp:
-    return expandLerpIntrinsic(Orig);
+    Result = expandLerpIntrinsic(Orig);
+    break;
   case Intrinsic::dx_length:
-    return expandLengthIntrinsic(Orig);
+    Result = expandLengthIntrinsic(Orig);
+    break;
   case Intrinsic::dx_normalize:
-    return expandNormalizeIntrinsic(Orig);
+    Result = expandNormalizeIntrinsic(Orig);
+    break;
   case Intrinsic::dx_sdot:
   case Intrinsic::dx_udot:
-    return expandIntegerDot(Orig, F.getIntrinsicID());
+    Result = expandIntegerDot(Orig, F.getIntrinsicID());
+    break;
+  }
+
+  if (Result) {
+    Orig->replaceAllUsesWith(Result);
+    Orig->eraseFromParent();
+    return true;
   }
   return false;
 }
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index 42df7c90cb337ea..ca372026141fbf3 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -87,6 +87,9 @@ static const char *getOverloadTypeName(OverloadKind Kind) {
 }
 
 static OverloadKind getOverloadKind(Type *Ty) {
+  if (!Ty)
+    return OverloadKind::VOID;
+
   Type::TypeID T = Ty->getTypeID();
   switch (T) {
   case Type::VoidTyID:
@@ -154,10 +157,8 @@ struct OpCodeProperty {
   llvm::SmallVector<OpOverload> Overloads;
   llvm::SmallVector<OpStage> Stages;
   llvm::SmallVector<OpAttribute> Attributes;
-  int OverloadParamIndex;        // parameter index which control the overload.
-                                 // When < 0, should be only 1 overload type.
-  unsigned NumOfParameters;      // Number of parameters include return value.
-  unsigned ParameterTableOffset; // Offset in ParameterTable.
+  int OverloadParamIndex; // parameter index which control the overload.
+                          // When < 0, should be only 1 overload type.
 };
 
 // Include getOpCodeClassName getOpCodeProperty, getOpCodeName and
@@ -208,35 +209,33 @@ static StructType *getHandleType(LLVMContext &Ctx) {
                                Ctx);
 }
 
-static Type *getTypeFromParameterKind(ParameterKind Kind, LLVMContext &Ctx,
-                                      Type *OverloadTy) {
+static Type *getTypeFromOpParamType(OpParamType Kind, LLVMContext &Ctx,
+                                    Type *OverloadTy) {
   switch (Kind) {
-  case ParameterKind::Void:
+  case OpParamType::VoidTy:
     return Type::getVoidTy(Ctx);
-  case ParameterKind::Half:
+  case OpParamType::HalfTy:
     return Type::getHalfTy(Ctx);
-  case ParameterKind::Float:
+  case OpParamType::FloatTy:
     return Type::getFloatTy(Ctx);
-  case ParameterKind::Double:
+  case OpParamType::DoubleTy:
     return Type::getDoubleTy(Ctx);
-  case ParameterKind::I1:
+  case OpParamType::Int1Ty:
     return Type::getInt1Ty(Ctx);
-  case ParameterKind::I8:
+  case OpParamType::Int8Ty:
     return Type::getInt8Ty(Ctx);
-  case ParameterKind::I16:
+  case OpParamType::Int16Ty:
     return Type::getInt16Ty(Ctx);
-  case ParameterKind::I32:
+  case OpParamType::Int32Ty:
     return Type::getInt32Ty(Ctx);
-  case ParameterKind::I64:
+  case OpParamType::Int64Ty:
     return Type::getInt64Ty(Ctx);
-  case ParameterKind::Overload:
+  case OpParamType::OverloadTy:
     return OverloadTy;
-  case ParameterKind::ResourceRet:
+  case OpParamType::ResRetTy:
     return getResRetType(OverloadTy, Ctx);
-  case ParameterKind::DXILHandle:
+  case OpParamType::HandleTy:
     return getHandleType(Ctx);
-  default:
-    break;
   }
   llvm_unreachable("Invalid parameter kind");
   return nullptr;
@@ -281,30 +280,34 @@ static ShaderKind getShaderKindEnum(Triple::EnvironmentType EnvType) {
       "Shader Kind Not Found - Invalid DXIL Environment Specified");
 }
 
+static SmallVector<Type *>
+getArgTypesFromOpParamTypes(ArrayRef<dxil::OpParamType> Types,
+                            LLVMContext &Context, Type *OverloadTy) {
+  SmallVector<Type *> ArgTys;
+  ArgTys.emplace_back(Type::getInt32Ty(Context));
+  for (dxil::OpParamType Ty : Types)
+    ArgTys.emplace_back(getTypeFromOpParamType(Ty, Context, OverloadTy));
+  return ArgTys;
+}
+
 /// Construct DXIL function type. This is the type of a function with
 /// the following prototype
 ///     OverloadType dx.op.<opclass>.<return-type>(int opcode, <param types>)
 /// <param-types> are constructed from types in Prop.
-static FunctionType *getDXILOpFunctionType(const OpCodeProperty *Prop,
+static FunctionType *getDXILOpFunctionType(dxil::OpCode OpCode,
                                            LLVMContext &Context,
                                            Type *OverloadTy) {
-  SmallVector<Type *> ArgTys;
-
-  const ParameterKind *ParamKinds = getOpCodeParameterKind(*Prop);
 
-  assert(Prop->NumOfParameters && "No return type?");
-  // Add return type of the function
-  Type *ReturnTy = getTypeFromParameterKind(ParamKinds[0], Context, OverloadTy);
-
-  // Add DXIL Opcode value type viz., Int32 as first argument
-  ArgTys.emplace_back(Type::getInt32Ty(Context));
-
-  // Add DXIL Operation parameter types as specified in DXIL properties
-  for (unsigned I = 1; I < Prop->NumOfParameters; ++I) {
-    ParameterKind Kind = ParamKinds[I];
-    ArgTys.emplace_back(getTypeFromParameterKind(Kind, Context, OverloadTy));
+  switch (OpCode) {
+#define DXIL_OP_FUNCTION_TYPE(OpCode, RetType, ...)                            \
+  case OpCode:                                                                 \
+    return FunctionType::get(                                                  \
+        getTypeFromOpParamType(RetType, Context, OverloadTy),                  \
+        getArgTypesFromOpParamTypes({__VA_ARGS__}, Context, OverloadTy),       \
+        /*isVarArg=*/false);
+#include "DXILOperation.inc"
   }
-  return FunctionType::get(ReturnTy, ArgTys, /*isVarArg=*/false);
+  llvm_unreachable("Invalid OpCode?");
 }
 
 /// Get index of the property from PropList valid for the most recent
@@ -369,7 +372,7 @@ Expected<CallInst *> DXILOpBuilder::tryCreateOp(dxil::OpCode OpCode,
     OverloadTy = Args[ArgIndex]->getType();
   }
   FunctionType *DXILOpFT =
-      getDXILOpFunctionType(Prop, M.getContext(), OverloadTy);
+      getDXILOpFunctionType(OpCode, M.getContext(), OverloadTy);
 
   std::optional<size_t> OlIndexOrErr =
       getPropIndex(ArrayRef(Prop->Overloads), DXILVersion);
@@ -379,11 +382,7 @@ Expected<CallInst *> DXILOpBuilder::tryCreateOp(dxil::OpCode OpCode,
 
   uint16_t ValidTyMask = Prop->Overloads[*OlIndexOrErr].ValidTys;
 
-  // If we don't have an overload type, use the function's return type. This is
-  // a bit of a hack, but it's necessary to get the type suffix on unoverloaded
-  // DXIL ops correct, like `dx.op.threadId.i32`.
-  OverloadKind Kind =
-      getOverloadKind(OverloadTy ? OverloadTy : DXILOpFT->getReturnType());
+  OverloadKind Kind = getOverloadKind(OverloadTy);
 
   // Check if the operation supports overload types and OverloadTy is valid
   // per the specified types for the operation
@@ -424,7 +423,7 @@ Expected<CallInst *> DXILOpBuilder::tryCreateOp(dxil::OpCode OpCode,
   return B.CreateCall(DXILFn, OpArgs);
 }
 
-CallInst *DXILOpBuilder::createOp(dxil::OpCode OpCode, ArrayRef<Value *> &Args,
+CallInst *DXILOpBuilder::createOp(dxil::OpCode OpCode, ArrayRef<Value *> Args,
                                   Type *RetTy) {
   Expected<CallInst *> Result = tryCreateOp(OpCode, Args, RetTy);
   if (Error E = Result.takeError())
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.h b/llvm/lib/Target/DirectX/DXILOpBuilder.h
index ff66f39a3ceb3d6..5d83357f7a2e944 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.h
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.h
@@ -33,25 +33,13 @@ class DXILOpBuilder {
 
   /// Create a call instruction for the given DXIL op. The arguments
   /// must be valid for an overload of the operation.
-  CallInst *createOp(dxil::OpCode Op, ArrayRef<Value *> &Args,
+  CallInst *createOp(dxil::OpCode Op, ArrayRef<Value *> Args,
                      Type *RetTy = nullptr);
 
-#define DXIL_OPCODE(Op, Name)                                                  \
-  CallInst *create##Name##Op(ArrayRef<Value *> &Args, Type *RetTy = nullptr) { \
-    return createOp(dxil::OpCode(Op), Args, RetTy);                            \
-  }
-#include "DXILOperation.inc"
-
   /// Try to create a call instruction for the given DXIL op. Fails if the
   /// overload is invalid.
   Expected<CallInst *> tryCreateOp(dxil::OpCode Op, ArrayRef<Value *> Args,
                                    Type *RetTy = nullptr);
-#define DXIL_OPCODE(Op, Name)                                                  \
-  Expected<CallInst *> tryCreate##Name##Op(ArrayRef<Value *> &Args,            \
-                                           Type *RetTy = nullptr) {            \
-    return tryCreateOp(dxil::OpCode(Op), Args, RetTy);                         \
-  }
-#include "DXILOperation.inc"
 
   /// Return the name of the given opcode.
   static const char *getOpCodeName(dxil::OpCode DXILOp);
diff --git a/llvm/lib/Target/DirectX/DXILResourceAnalysis.h b/llvm/lib/Target/DirectX/DXILResourceAnalysis.h
index 3a2b8a9fd39d591..26d9237d51b49cb 100644
--- a/llvm/lib/Target/DirectX/DXILResourceAnalysis.h
+++ b/llvm/lib/Target/DirectX/DXILResourceAnalysis.h
@@ -57,6 +57,10 @@ class DXILResourceMDWrapper : public ModulePass {
   /// Calculate the DXILResource for the module.
   bool runOnModule(Module &M) override;
 
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+
   void print(raw_ostream &O, const Module *M = nullptr) const override;
 };
 } // namespace llvm
diff --git a/llvm/lib/Target/DirectX/DirectX.h b/llvm/lib/Target/DirectX/DirectX.h
index d056ae2bc488e7a..de6c6f2db5b2448 100644
--- a/llvm/lib/Target/DirectX/DirectX.h
+++ b/llvm/lib/Target/DirectX/DirectX.h
@@ -63,6 +63,13 @@ void initializeDXContainerGlobalsPass(PassRegistry &);
 
 /// Pass for generating DXContainer part globals.
 ModulePass *createDXContainerGlobalsPass();
+
+/// Initializer for DXILFinalizeLinkage pass.
+void initializeDXILFinalizeLinkageLegacyPass(PassRegistry &);
+
+/// Pass to finalize linkage of functions.
+ModulePass *createDXILFinalizeLinkageLegacyPass();
+
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_DIRECTX_DIRECTX_H
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index 92bd69b69684f06..2c2b86e7193382e 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -48,6 +48,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeDirectXTarget() {
   initializeDXILTranslateMetadataPass(*PR);
   initializeDXILResourceMDWrapperPass(*PR);
   initializeShaderFlagsAnalysisWrapperPass(*PR);
+  initializeDXILFinalizeLinkageLegacyPass(*PR);
 }
 
 class DXILTargetObjectFile : public TargetLoweringObjectFile {
@@ -79,6 +80,7 @@ class DirectXPassConfig : public TargetPassConfig {
   void addCodeGenPrepare() override {
     addPass(createDXILIntrinsicExpansionLegacyPass());
     addPass(createDXILOpLoweringLegacyPass());
+    addPass(createDXILFinalizeLinkageLegacyPass());
     addPass(createDXILTranslateMetadataPass());
     addPass(createDXILPrepareModulePass());
   }
diff --git a/llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp b/llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp
index a2230289ae69c7d..59c882bf37afaa7 100644
--- a/llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp
@@ -249,9 +249,7 @@ void HexagonCopyHoisting::moveCopyInstr(MachineBasicBlock *DestBB,
   DestBB->splice(FirstTI, MI->getParent(), MI);
 
   addMItoCopyList(MI);
-  for (auto I = std::next(DestBB->succ_begin()), E = DestBB->succ_end(); I != E;
-       ++I) {
-    MachineBasicBlock *SuccBB = *I;
+  for (MachineBasicBlock *SuccBB : drop_begin(DestBB->successors())) {
     auto &BBCopyInst = CopyMIList[SuccBB->getNumber()];
     MachineInstr *SuccMI = BBCopyInst[Key];
     SuccMI->eraseFromParent();
diff --git a/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp b/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
index 651ccc2db9ba24c..92574a16c4f9318 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
@@ -86,11 +86,9 @@ bool HexagonGenMemAbsolute::runOnMachineFunction(MachineFunction &Fn) {
       getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
 
   // Loop over all of the basic blocks
-  for (MachineFunction::iterator MBBb = Fn.begin(), MBBe = Fn.end();
-       MBBb != MBBe; ++MBBb) {
-    MachineBasicBlock *MBB = &*MBBb;
+  for (MachineBasicBlock &MBB : Fn) {
     // Traverse the basic block
-    for (MachineBasicBlock::iterator MII = MBB->begin(); MII != MBB->end();
+    for (MachineBasicBlock::iterator MII = MBB.begin(); MII != MBB.end();
          ++MII) {
       MachineInstr *MI = &*MII;
       int Opc = MI->getOpcode();
@@ -205,7 +203,7 @@ bool HexagonGenMemAbsolute::runOnMachineFunction(MachineFunction &Fn) {
 
       LLVM_DEBUG(dbgs() << "Replaced with " << *MIB << "\n");
       // Erase the instructions that got replaced.
-      MII = MBB->erase(MI);
+      MII = MBB.erase(MI);
       --MII;
       NextMI->getParent()->erase(NextMI);
     }
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index 2e66c81c3324492..273bd7b2b361e70 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -21,7 +21,7 @@
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/InstSimplifyFolder.h"
 #include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 44d684331b3eb46..50c6c263e966b54 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -5619,8 +5619,9 @@ bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
 
   // On LA64, atomic store operations with IntegerBitWidth of 32 and 64 do not
   // require fences beacuse we can use amswap_db.[w/d].
-  if (isa<StoreInst>(I)) {
-    unsigned Size = I->getOperand(0)->getType()->getIntegerBitWidth();
+  Type *Ty = I->getOperand(0)->getType();
+  if (isa<StoreInst>(I) && Ty->isIntegerTy()) {
+    unsigned Size = Ty->getIntegerBitWidth();
     return (Size == 8 || Size == 16);
   }
 
diff --git a/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp b/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
index ae50b7a6f923e3a..47071d29c2cd44e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
@@ -381,6 +381,14 @@ static unsigned getNewOpc(unsigned Op, bool isLarge) {
     return isLarge ? LoongArch::FLDX_S : LoongArch::FLD_S;
   case LoongArch::FLD_D:
     return isLarge ? LoongArch::FLDX_D : LoongArch::FLD_D;
+  case LoongArch::VLD:
+    return isLarge ? LoongArch::VLDX : LoongArch::VLD;
+  case LoongArch::XVLD:
+    return isLarge ? LoongArch::XVLDX : LoongArch::XVLD;
+  case LoongArch::VLDREPL_B:
+    return LoongArch::VLDREPL_B;
+  case LoongArch::XVLDREPL_B:
+    return LoongArch::XVLDREPL_B;
   case LoongArch::ST_B:
     return isLarge ? LoongArch::STX_B : LoongArch::ST_B;
   case LoongArch::ST_H:
@@ -395,6 +403,10 @@ static unsigned getNewOpc(unsigned Op, bool isLarge) {
     return isLarge ? LoongArch::FSTX_S : LoongArch::FST_S;
   case LoongArch::FST_D:
     return isLarge ? LoongArch::FSTX_D : LoongArch::FST_D;
+  case LoongArch::VST:
+    return isLarge ? LoongArch::VSTX : LoongArch::VST;
+  case LoongArch::XVST:
+    return isLarge ? LoongArch::XVSTX : LoongArch::XVST;
   default:
     llvm_unreachable("Unexpected opcode for replacement");
   }
@@ -444,6 +456,12 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
     default:
       LLVM_DEBUG(dbgs() << "Not a load or store instruction: " << UseMI);
       return false;
+    case LoongArch::VLDREPL_B:
+    case LoongArch::XVLDREPL_B:
+      // We can't do this for large pattern.
+      if (Last)
+        return false;
+      [[fallthrough]];
     case LoongArch::LD_B:
     case LoongArch::LD_H:
     case LoongArch::LD_W:
@@ -455,6 +473,8 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
     case LoongArch::LDPTR_D:
     case LoongArch::FLD_S:
     case LoongArch::FLD_D:
+    case LoongArch::VLD:
+    case LoongArch::XVLD:
     case LoongArch::ST_B:
     case LoongArch::ST_H:
     case LoongArch::ST_W:
@@ -462,7 +482,9 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
     case LoongArch::STPTR_W:
     case LoongArch::STPTR_D:
     case LoongArch::FST_S:
-    case LoongArch::FST_D: {
+    case LoongArch::FST_D:
+    case LoongArch::VST:
+    case LoongArch::XVST: {
       if (UseMI.getOperand(1).isFI())
         return false;
       // Register defined by Lo should not be the value register.
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index 4d6a00c14a3575c..19b6bf0da22a60a 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -47,13 +47,6 @@ struct MipsRelocationEntry {
   }
 };
 
-#ifndef NDEBUG
-raw_ostream &operator<<(raw_ostream &OS, const MipsRelocationEntry &RHS) {
-  RHS.print(OS);
-  return OS;
-}
-#endif
-
 class MipsELFObjectWriter : public MCELFObjectTargetWriter {
 public:
   MipsELFObjectWriter(uint8_t OSABI, bool HasRelocationAddend, bool Is64);
@@ -115,17 +108,11 @@ static InputIt find_best(InputIt First, InputIt Last, UnaryPredicate Predicate,
   for (InputIt I = First; I != Last; ++I) {
     unsigned Matched = Predicate(*I);
     if (Matched != FindBest_NoMatch) {
-      LLVM_DEBUG(dbgs() << std::distance(First, I) << " is a match (";
-                 I->print(dbgs()); dbgs() << ")\n");
-      if (Best == Last || BetterThan(*I, *Best)) {
-        LLVM_DEBUG(dbgs() << ".. and it beats the last one\n");
+      if (Best == Last || BetterThan(*I, *Best))
         Best = I;
-      }
     }
-    if (Matched == FindBest_PerfectMatch) {
-      LLVM_DEBUG(dbgs() << ".. and it is unbeatable\n");
+    if (Matched == FindBest_PerfectMatch)
       break;
-    }
   }
 
   return Best;
@@ -179,10 +166,9 @@ static FindBestPredicateResult isMatchingReloc(const MipsRelocationEntry &X,
                                                const ELFRelocationEntry &R,
                                                unsigned MatchingType) {
   if (X.R.Type == MatchingType && X.R.OriginalSymbol == R.OriginalSymbol) {
-    if (!X.Matched &&
-        X.R.OriginalAddend == R.OriginalAddend)
+    if (!X.Matched && X.R.Addend == R.Addend)
       return FindBest_PerfectMatch;
-    else if (X.R.OriginalAddend >= R.OriginalAddend)
+    else if (X.R.Addend >= R.Addend)
       return FindBest_Match;
   }
   return FindBest_NoMatch;
@@ -196,20 +182,11 @@ static FindBestPredicateResult isMatchingReloc(const MipsRelocationEntry &X,
 /// - It is not already involved in a match.
 static bool compareMatchingRelocs(const MipsRelocationEntry &Candidate,
                                   const MipsRelocationEntry &PreviousBest) {
-  if (Candidate.R.OriginalAddend != PreviousBest.R.OriginalAddend)
-    return Candidate.R.OriginalAddend < PreviousBest.R.OriginalAddend;
+  if (Candidate.R.Addend != PreviousBest.R.Addend)
+    return Candidate.R.Addend < PreviousBest.R.Addend;
   return PreviousBest.Matched && !Candidate.Matched;
 }
 
-#ifndef NDEBUG
-/// Print all the relocations.
-template <class Container>
-static void dumpRelocs(const char *Prefix, const Container &Relocs) {
-  for (const auto &R : Relocs)
-    dbgs() << Prefix << R << "\n";
-}
-#endif
-
 MipsELFObjectWriter::MipsELFObjectWriter(uint8_t OSABI,
                                          bool HasRelocationAddend, bool Is64)
     : MCELFObjectTargetWriter(Is64, OSABI, ELF::EM_MIPS, HasRelocationAddend) {}
@@ -448,8 +425,6 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm,
   std::list<MipsRelocationEntry> Sorted;
   std::list<ELFRelocationEntry> Remainder;
 
-  LLVM_DEBUG(dumpRelocs("R: ", Relocs));
-
   // Separate the movable relocations (AHL relocations using the high bits) from
   // the immobile relocations (everything else). This does not preserve high/low
   // matches that already existed in the input.
@@ -459,8 +434,6 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm,
                });
 
   for (auto &R : Remainder) {
-    LLVM_DEBUG(dbgs() << "Matching: " << R << "\n");
-
     unsigned MatchingType = getMatchingLoType(R);
     assert(MatchingType != ELF::R_MIPS_NONE &&
            "Wrong list for reloc that doesn't need a match");
@@ -494,8 +467,6 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm,
     Sorted.insert(InsertionPoint, R)->Matched = true;
   }
 
-  LLVM_DEBUG(dumpRelocs("S: ", Sorted));
-
   assert(Relocs.size() == Sorted.size() && "Some relocs were not consumed");
 
   // Overwrite the original vector with the sorted elements.
diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp
index 3485825b77627ee..7d8278c8ca3cf55 100644
--- a/llvm/lib/Target/Mips/MipsFastISel.cpp
+++ b/llvm/lib/Target/Mips/MipsFastISel.cpp
@@ -1608,8 +1608,8 @@ bool MipsFastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         }
         emitInst(Mips::SLL, TempReg[0]).addReg(SrcReg).addImm(8);
         emitInst(Mips::SRL, TempReg[1]).addReg(SrcReg).addImm(8);
-        emitInst(Mips::OR, TempReg[2]).addReg(TempReg[0]).addReg(TempReg[1]);
-        emitInst(Mips::ANDi, DestReg).addReg(TempReg[2]).addImm(0xFFFF);
+        emitInst(Mips::ANDi, TempReg[2]).addReg(TempReg[1]).addImm(0xFF);
+        emitInst(Mips::OR, DestReg).addReg(TempReg[0]).addReg(TempReg[2]);
         updateValueMap(II, DestReg);
         return true;
       }
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 43a3fbf4d1306a9..73791102fc04dee 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -429,29 +429,50 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
 
   auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                     LegalizeAction NoF16Action) {
-    setOperationAction(Op, VT, STI.allowFP16Math() ? Action : NoF16Action);
+    bool IsOpSupported = STI.allowFP16Math();
+    switch (Op) {
+    // Several FP16 instructions are available on sm_80 only.
+    case ISD::FMINNUM:
+    case ISD::FMAXNUM:
+    case ISD::FMAXNUM_IEEE:
+    case ISD::FMINNUM_IEEE:
+    case ISD::FMAXIMUM:
+    case ISD::FMINIMUM:
+      IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
+      break;
+    }
+    setOperationAction(Op, VT, IsOpSupported ? Action : NoF16Action);
   };
 
   auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                     LegalizeAction NoBF16Action) {
     bool IsOpSupported = STI.hasBF16Math();
-    // Few instructions are available on sm_90 only
-    switch(Op) {
-      case ISD::FADD:
-      case ISD::FMUL:
-      case ISD::FSUB:
-      case ISD::SELECT:
-      case ISD::SELECT_CC:
-      case ISD::SETCC:
-      case ISD::FEXP2:
-      case ISD::FCEIL:
-      case ISD::FFLOOR:
-      case ISD::FNEARBYINT:
-      case ISD::FRINT:
-      case ISD::FROUNDEVEN:
-      case ISD::FTRUNC:
-        IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
-        break;
+    switch (Op) {
+    // Several BF16 instructions are available on sm_90 only.
+    case ISD::FADD:
+    case ISD::FMUL:
+    case ISD::FSUB:
+    case ISD::SELECT:
+    case ISD::SELECT_CC:
+    case ISD::SETCC:
+    case ISD::FEXP2:
+    case ISD::FCEIL:
+    case ISD::FFLOOR:
+    case ISD::FNEARBYINT:
+    case ISD::FRINT:
+    case ISD::FROUNDEVEN:
+    case ISD::FTRUNC:
+      IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
+      break;
+    // Several BF16 instructions are available on sm_80 only.
+    case ISD::FMINNUM:
+    case ISD::FMAXNUM:
+    case ISD::FMAXNUM_IEEE:
+    case ISD::FMINNUM_IEEE:
+    case ISD::FMAXIMUM:
+    case ISD::FMINIMUM:
+      IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
+      break;
     }
     setOperationAction(
         Op, VT, IsOpSupported ? Action : NoBF16Action);
@@ -648,6 +669,8 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
 
   // TRAP can be lowered to PTX trap
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
+  // DEBUGTRAP can be lowered to PTX brkpt
+  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
 
   // Register custom handling for vector loads/stores
   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
@@ -836,26 +859,23 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
       AddPromotedToType(Op, MVT::bf16, MVT::f32);
   }
 
-  // max.f16, max.f16x2 and max.NaN are supported on sm_80+.
-  auto GetMinMaxAction = [&](LegalizeAction NotSm80Action) {
-    bool IsAtLeastSm80 = STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
-    return IsAtLeastSm80 ? Legal : NotSm80Action;
-  };
   for (const auto &Op : {ISD::FMINNUM, ISD::FMAXNUM}) {
-    setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Promote), Promote);
     setOperationAction(Op, MVT::f32, Legal);
     setOperationAction(Op, MVT::f64, Legal);
-    setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
+    setFP16OperationAction(Op, MVT::f16, Legal, Promote);
+    setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
     setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
     if (getOperationAction(Op, MVT::bf16) == Promote)
       AddPromotedToType(Op, MVT::bf16, MVT::f32);
   }
+  bool SupportsF32MinMaxNaN =
+      STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
   for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
-    setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Expand), Expand);
-    setFP16OperationAction(Op, MVT::bf16, Legal, Expand);
-    setOperationAction(Op, MVT::f32, GetMinMaxAction(Expand));
-    setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
+    setOperationAction(Op, MVT::f32, SupportsF32MinMaxNaN ? Legal : Expand);
+    setFP16OperationAction(Op, MVT::f16, Legal, Expand);
+    setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
+    setBF16OperationAction(Op, MVT::bf16, Legal, Expand);
     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
   }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index b57c86fcf697cdb..a6dfb704e38d2e2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -723,6 +723,35 @@ let hasSideEffects = false in {
 
   defm CVT_f16x2 : CVT_FROM_FLOAT_V2_SM80<"f16x2", Int32Regs>;
   defm CVT_bf16x2 : CVT_FROM_FLOAT_V2_SM80<"bf16x2", Int32Regs>;
+
+  // FP8 conversions.
+  multiclass CVT_TO_F8X2<string F8Name> {
+    def _f32 :
+      NVPTXInst<(outs Int16Regs:$dst),
+                (ins Float32Regs:$src1, Float32Regs:$src2, CvtMode:$mode),
+                !strconcat("cvt${mode:base}.satfinite${mode:relu}.",
+                F8Name, "x2.f32 \t$dst, $src1, $src2;"), []>,
+      Requires<[hasPTX<81>, hasSM<89>]>;
+    def _f16x2 :
+      NVPTXInst<(outs Int16Regs:$dst),
+                (ins Int32Regs:$src, CvtMode:$mode),
+                !strconcat("cvt${mode:base}.satfinite${mode:relu}.",
+                F8Name, "x2.f16x2 \t$dst, $src;"), []>,
+      Requires<[hasPTX<81>, hasSM<89>]>;
+  }
+
+  defm CVT_e4m3x2 : CVT_TO_F8X2<"e4m3">;
+  defm CVT_e5m2x2 : CVT_TO_F8X2<"e5m2">;
+
+  class CVT_f16x2_fp8<string F8Name> :
+    NVPTXInst<(outs Int32Regs:$dst),
+              (ins Int16Regs:$src, CvtMode:$mode),
+              !strconcat("cvt${mode:base}${mode:relu}.f16x2.",
+              F8Name, "x2 \t$dst, $src;"), []>,
+    Requires<[hasPTX<81>, hasSM<89>]>;
+
+  def CVT_f16x2_e4m3x2 : CVT_f16x2_fp8<"e4m3">;
+  def CVT_f16x2_e5m2x2 : CVT_f16x2_fp8<"e5m2">;
 }
 
 //-----------------------------------
@@ -3845,6 +3874,8 @@ def Callseq_End :
 // Emit an `exit` as well to convey to ptxas that `trap` exits the CFG.
 // This won't be necessary in a future version of ptxas.
 def trapinst : NVPTXInst<(outs), (ins), "trap; exit;", [(trap)]>;
+// brkpt instruction
+def debugtrapinst : NVPTXInst<(outs), (ins), "brkpt;", [(debugtrap)]>;
 
 // Call prototype wrapper
 def SDTCallPrototype : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 887951b55fb3b70..a9116e15c3671be 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1524,6 +1524,33 @@ def : Pat<(int_nvvm_f2h_rn_ftz Float32Regs:$a),
 def : Pat<(int_nvvm_f2h_rn Float32Regs:$a),
           (CVT_f16_f32 Float32Regs:$a, CvtRN)>;
 
+def : Pat<(int_nvvm_ff_to_e4m3x2_rn Float32Regs:$a, Float32Regs:$b),
+          (CVT_e4m3x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>;
+def : Pat<(int_nvvm_ff_to_e4m3x2_rn_relu Float32Regs:$a, Float32Regs:$b),
+          (CVT_e4m3x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN_RELU)>;
+def : Pat<(int_nvvm_ff_to_e5m2x2_rn Float32Regs:$a, Float32Regs:$b),
+          (CVT_e5m2x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>;
+def : Pat<(int_nvvm_ff_to_e5m2x2_rn_relu Float32Regs:$a, Float32Regs:$b),
+          (CVT_e5m2x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN_RELU)>;
+
+def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn Int32Regs:$a),
+          (CVT_e4m3x2_f16x2 Int32Regs:$a, CvtRN)>;
+def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn_relu Int32Regs:$a),
+          (CVT_e4m3x2_f16x2 Int32Regs:$a, CvtRN_RELU)>;
+def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn Int32Regs:$a),
+          (CVT_e5m2x2_f16x2 Int32Regs:$a, CvtRN)>;
+def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn_relu Int32Regs:$a),
+          (CVT_e5m2x2_f16x2 Int32Regs:$a, CvtRN_RELU)>;
+
+def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn Int16Regs:$a),
+          (CVT_f16x2_e4m3x2 Int16Regs:$a, CvtRN)>;
+def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn_relu Int16Regs:$a),
+          (CVT_f16x2_e4m3x2 Int16Regs:$a, CvtRN_RELU)>;
+def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn Int16Regs:$a),
+          (CVT_f16x2_e5m2x2 Int16Regs:$a, CvtRN)>;
+def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn_relu Int16Regs:$a),
+          (CVT_f16x2_e5m2x2 Int16Regs:$a, CvtRN_RELU)>;
+
 //
 // Bitcast
 //
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 8ff9f5a5a991e08..459a96eca1ff202 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -3388,17 +3388,15 @@ static void updateForAIXShLibTLSModelOpt(TLSModel::Model &Model,
     // global variables (global variables taken as the first parameter to
     // Intrinsic::threadlocal_address).
     const Function &Func = DAG.getMachineFunction().getFunction();
-    for (Function::const_iterator BI = Func.begin(), BE = Func.end(); BI != BE;
-         ++BI)
-      for (BasicBlock::const_iterator II = BI->begin(), IE = BI->end();
-           II != IE; ++II)
-        if (II->getOpcode() == Instruction::Call)
-          if (const CallInst *CI = dyn_cast<const CallInst>(&*II))
+    for (const BasicBlock &BB : Func)
+      for (const Instruction &I : BB)
+        if (I.getOpcode() == Instruction::Call)
+          if (const CallInst *CI = dyn_cast<const CallInst>(&I))
             if (Function *CF = CI->getCalledFunction())
               if (CF->isDeclaration() &&
                   CF->getIntrinsicID() == Intrinsic::threadlocal_address)
                 if (const GlobalValue *GV =
-                        dyn_cast<GlobalValue>(II->getOperand(0))) {
+                        dyn_cast<GlobalValue>(I.getOperand(0))) {
                   TLSModel::Model GVModel = TM.getTLSModel(GV);
                   if (GVModel == TLSModel::LocalDynamic)
                     TLSGV.insert(GV);
@@ -9515,10 +9513,8 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     } else {
       // We may lose precision, so we have to use XXSPLTI32DX.
 
-      uint32_t Hi =
-          (uint32_t)((APSplatBits.getZExtValue() & 0xFFFFFFFF00000000LL) >> 32);
-      uint32_t Lo =
-          (uint32_t)(APSplatBits.getZExtValue() & 0xFFFFFFFF);
+      uint32_t Hi = Hi_32(APSplatBits.getZExtValue());
+      uint32_t Lo = Lo_32(APSplatBits.getZExtValue());
       SDValue SplatNode = DAG.getUNDEF(MVT::v2i64);
 
       if (!Hi || !Lo)
@@ -9662,8 +9658,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
                                   dl);
 
   // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
-  int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >>
-                    (32-SplatBitSize));
+  int32_t SextVal = SignExtend32(SplatBits, SplatBitSize);
   if (SextVal >= -16 && SextVal <= 15)
     return getCanonicalConstSplat(SextVal, SplatSize, Op.getValueType(), DAG,
                                   dl);
diff --git a/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp b/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
index 309938accdf4c3b..daf6a0e65d54b24 100644
--- a/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
@@ -170,8 +170,9 @@ void PPCMergeStringPool::collectCandidateConstants(Module &M) {
     LLVM_DEBUG(dbgs() << "hasInitializer() " << Global.hasInitializer()
                       << "\n");
 
-    // We can only pool constants.
-    if (!Global.isConstant() || !Global.hasInitializer())
+    // We can only pool non-thread-local constants.
+    if (!Global.isConstant() || !Global.hasInitializer() ||
+        Global.isThreadLocal())
       continue;
 
     // If a global constant has a section we do not try to pool it because
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index 763b6edb1c09fba..6ce345dd44138dc 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -500,7 +500,12 @@ void PPCPassConfig::addIRPasses() {
 }
 
 bool PPCPassConfig::addPreISel() {
-  if (EnableGlobalMerge)
+  // The GlobalMerge pass is intended to be on by default on AIX.
+  // Specifying the command line option overrides the AIX default.
+  if ((EnableGlobalMerge.getNumOccurrences() > 0)
+          ? EnableGlobalMerge
+          : (TM->getTargetTriple().isOSAIX() &&
+             getOptLevel() != CodeGenOptLevel::None))
     addPass(
         createGlobalMergePass(TM, GlobalMergeMaxOffset, false, false, true));
 
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index a288e7d884d311d..f299e823aa3f500 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -1483,10 +1483,6 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     if (isRV64())
       return generateImmOutOfRangeError(Operands, ErrorInfo, 1, (1 << 6) - 1);
     return generateImmOutOfRangeError(Operands, ErrorInfo, 1, (1 << 5) - 1);
-  case Match_InvalidUImmLog2XLenHalf:
-    if (isRV64())
-      return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 5) - 1);
-    return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 4) - 1);
   case Match_InvalidUImm1:
     return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 1) - 1);
   case Match_InvalidUImm2:
@@ -1913,7 +1909,7 @@ ParseStatus RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) {
     if (getParser().parseIdentifier(Identifier))
       return ParseStatus::Failure;
 
-    auto SysReg = RISCVSysReg::lookupSysRegByName(Identifier);
+    const auto *SysReg = RISCVSysReg::lookupSysRegByName(Identifier);
     if (!SysReg)
       SysReg = RISCVSysReg::lookupSysRegByAltName(Identifier);
     if (!SysReg)
@@ -1923,8 +1919,24 @@ ParseStatus RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) {
 
     // Accept a named Sys Reg if the required features are present.
     if (SysReg) {
-      if (!SysReg->haveRequiredFeatures(getSTI().getFeatureBits()))
-        return Error(S, "system register use requires an option to be enabled");
+      const auto &FeatureBits = getSTI().getFeatureBits();
+      if (!SysReg->haveRequiredFeatures(FeatureBits)) {
+        const auto *Feature = llvm::find_if(RISCVFeatureKV, [&](auto Feature) {
+          return SysReg->FeaturesRequired[Feature.Value];
+        });
+        auto ErrorMsg = std::string("system register '") + SysReg->Name + "' ";
+        if (SysReg->isRV32Only && FeatureBits[RISCV::Feature64Bit]) {
+          ErrorMsg += "is RV32 only";
+          if (Feature != std::end(RISCVFeatureKV))
+            ErrorMsg += " and ";
+        }
+        if (Feature != std::end(RISCVFeatureKV)) {
+          ErrorMsg +=
+              "requires '" + std::string(Feature->Key) + "' to be enabled";
+        }
+
+        return Error(S, ErrorMsg);
+      }
       Operands.push_back(
           RISCVOperand::createSysReg(Identifier, S, SysReg->Encoding));
       return ParseStatus::Success;
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 7491af1aae2a5a3..1978c07f0cafe2e 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -74,7 +74,9 @@ static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,
   LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
     return ST.hasVInstructions() &&
            (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
-            ST.getELen() == 64);
+            ST.getELen() == 64) &&
+           (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||
+            Query.Types[TypeIdx].getScalarSizeInBits() == 32);
   };
   return all(typeInSet(TypeIdx, PtrVecTys), P);
 }
@@ -127,6 +129,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
   const LLT nxv2p0 = LLT::scalable_vector(2, p0);
   const LLT nxv4p0 = LLT::scalable_vector(4, p0);
   const LLT nxv8p0 = LLT::scalable_vector(8, p0);
+  const LLT nxv16p0 = LLT::scalable_vector(16, p0);
 
   using namespace TargetOpcode;
 
@@ -137,7 +140,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
                         nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
                         nxv1s64,  nxv2s64, nxv4s64, nxv8s64};
 
-  auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0};
+  auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};
 
   getActionDefinitionsBuilder({G_ADD, G_SUB, G_AND, G_OR, G_XOR})
       .legalFor({s32, sXLen})
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index e278fa3fe31764e..a439e75a3d5af43 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -754,6 +754,14 @@ def HasStdExtZvbc : Predicate<"Subtarget->hasStdExtZvbc()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvbc),
                         "'Zvbc' (Vector Carryless Multiplication)">;
 
+def FeatureStdExtZvbc32e
+    : RISCVExperimentalExtension<"zvbc32e", 0, 7,
+                                 "'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements)">;
+
+def HasStdExtZvbcOrZvbc32e : Predicate<"Subtarget->hasStdExtZvbc() || Subtarget->hasStdExtZvbc32e()">,
+                             AssemblerPredicate<(any_of FeatureStdExtZvbc, FeatureStdExtZvbc32e),
+                             "'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication)">;
+
 def FeatureStdExtZvkg
     : RISCVExtension<"zvkg", 1, 0,
                      "'Zvkg' (Vector GCM instructions for Cryptography)">,
@@ -762,6 +770,14 @@ def HasStdExtZvkg : Predicate<"Subtarget->hasStdExtZvkg()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvkg),
                         "'Zvkg' (Vector GCM instructions for Cryptography)">;
 
+def FeatureStdExtZvkgs
+    : RISCVExperimentalExtension<"zvkgs", 0, 7,
+                                 "'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)",
+                                 [FeatureStdExtZvkg]>;
+def HasStdExtZvkgs : Predicate<"Subtarget->hasStdExtZvkgs()">,
+                     AssemblerPredicate<(all_of FeatureStdExtZvkgs),
+                        "'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)">;
+
 def FeatureStdExtZvkned
     : RISCVExtension<"zvkned", 1, 0,
                      "'Zvkned' (Vector AES Encryption & Decryption (Single Round))">,
@@ -780,8 +796,7 @@ def HasStdExtZvknha : Predicate<"Subtarget->hasStdExtZvknha()">,
 
 def FeatureStdExtZvknhb
     : RISCVExtension<"zvknhb", 1, 0,
-                     "'Zvknhb' (Vector SHA-2 (SHA-256 and SHA-512))",
-                     [FeatureStdExtZve64x]>,
+                     "'Zvknhb' (Vector SHA-2 (SHA-256 and SHA-512))">,
       RISCVExtensionBitmask<0, 56>;
 def HasStdExtZvknhb : Predicate<"Subtarget->hasStdExtZvknhb()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvknhb),
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 35681c620eed51e..11210e6cec177ff 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -177,7 +177,8 @@ static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
                             RISCVMatInt::InstSeq &Seq) {
   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, VT);
   for (const RISCVMatInt::Inst &Inst : Seq) {
-    SDValue SDImm = CurDAG->getTargetConstant(Inst.getImm(), DL, VT);
+    SDValue SDImm =
+        CurDAG->getSignedConstant(Inst.getImm(), DL, VT, /*isTarget=*/true);
     SDNode *Result = nullptr;
     switch (Inst.getOpndKind()) {
     case RISCVMatInt::Imm:
@@ -208,10 +209,10 @@ static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
 
   // Use a rematerializable pseudo instruction for short sequences if enabled.
   if (Seq.size() == 2 && UsePseudoMovImm)
-    return SDValue(
-        CurDAG->getMachineNode(RISCV::PseudoMovImm, DL, VT,
-                               CurDAG->getTargetConstant(Imm, DL, VT)),
-        0);
+    return SDValue(CurDAG->getMachineNode(RISCV::PseudoMovImm, DL, VT,
+                                          CurDAG->getSignedConstant(
+                                              Imm, DL, VT, /*isTarget=*/true)),
+                   0);
 
   // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
   // worst an LUI+ADDIW. This will require an extra register, but avoids a
@@ -670,9 +671,9 @@ bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
 
   unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
 
-  SDNode *BinOp =
-      CurDAG->getMachineNode(BinOpc, DL, VT, Shift.getOperand(0),
-                             CurDAG->getTargetConstant(ShiftedVal, DL, VT));
+  SDNode *BinOp = CurDAG->getMachineNode(
+      BinOpc, DL, VT, Shift.getOperand(0),
+      CurDAG->getSignedConstant(ShiftedVal, DL, VT, /*isTarget=*/true));
   SDNode *SLLI =
       CurDAG->getMachineNode(ShOpc, DL, VT, SDValue(BinOp, 0),
                              CurDAG->getTargetConstant(ShAmt, DL, VT));
@@ -802,7 +803,8 @@ bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
 
   EVT Ty = Ld->getOffset().getValueType();
   SDValue Ops[] = {Ld->getBasePtr(),
-                   CurDAG->getTargetConstant(Offset >> Shift, SDLoc(Node), Ty),
+                   CurDAG->getSignedConstant(Offset >> Shift, SDLoc(Node), Ty,
+                                             /*isTarget=*/true),
                    CurDAG->getTargetConstant(Shift, SDLoc(Node), Ty),
                    Ld->getChain()};
   SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(Node), Ld->getValueType(0),
@@ -1164,6 +1166,15 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     }
 
     unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
+    if (Subtarget->hasVendorXTHeadBb()) {
+      SDNode *THEXTU = CurDAG->getMachineNode(
+          RISCV::TH_EXTU, DL, VT, N0->getOperand(0),
+          CurDAG->getTargetConstant(TrailingOnes - 1, DL, VT),
+          CurDAG->getTargetConstant(ShAmt, DL, VT));
+      ReplaceNode(Node, THEXTU);
+      return;
+    }
+
     SDNode *SLLI =
         CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
                                CurDAG->getTargetConstant(LShAmt, DL, VT));
@@ -2477,8 +2488,8 @@ bool RISCVDAGToDAGISel::SelectFrameAddrRegImm(SDValue Addr, SDValue &Base,
     if (isInt<12>(CVal)) {
       Base = CurDAG->getTargetFrameIndex(FIN->getIndex(),
                                          Subtarget->getXLenVT());
-      Offset = CurDAG->getTargetConstant(CVal, SDLoc(Addr),
-                                         Subtarget->getXLenVT());
+      Offset = CurDAG->getSignedConstant(
+          CVal, SDLoc(Addr), Subtarget->getXLenVT(), /*isTarget=*/true);
       return true;
     }
   }
@@ -2514,7 +2525,7 @@ static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
     } else {
       Base = CurDAG->getRegister(RISCV::X0, VT);
     }
-    Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
+    Offset = CurDAG->getSignedConstant(Lo12, DL, VT, /*isTarget=*/true);
     return true;
   }
 
@@ -2534,7 +2545,7 @@ static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
   assert(!Seq.empty() && "Expected more instructions in sequence");
 
   Base = selectImmSeq(CurDAG, DL, VT, Seq);
-  Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
+  Offset = CurDAG->getSignedConstant(Lo12, DL, VT, /*isTarget=*/true);
   return true;
 }
 
@@ -2659,7 +2670,7 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
 
       if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
-      Offset = CurDAG->getTargetConstant(CVal, DL, VT);
+      Offset = CurDAG->getSignedConstant(CVal, DL, VT, /*isTarget=*/true);
       return true;
     }
   }
@@ -2676,10 +2687,11 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
     if (isInt<12>(CVal / 2) && isInt<12>(CVal - CVal / 2)) {
       int64_t Adj = CVal < 0 ? -2048 : 2047;
       Base = SDValue(
-          CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0),
-                                 CurDAG->getTargetConstant(Adj, DL, VT)),
+          CurDAG->getMachineNode(
+              RISCV::ADDI, DL, VT, Addr.getOperand(0),
+              CurDAG->getSignedConstant(Adj, DL, VT, /*isTarget=*/true)),
           0);
-      Offset = CurDAG->getTargetConstant(CVal - Adj, DL, VT);
+      Offset = CurDAG->getSignedConstant(CVal - Adj, DL, VT, /*isTarget=*/true);
       return true;
     }
 
@@ -2732,7 +2744,7 @@ bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
 
       if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
-      Offset = CurDAG->getTargetConstant(CVal, DL, VT);
+      Offset = CurDAG->getSignedConstant(CVal, DL, VT, /*isTarget=*/true);
       return true;
     }
   }
@@ -2750,9 +2762,10 @@ bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
       int64_t AdjustedOffset = CVal - Adj;
       Base = SDValue(CurDAG->getMachineNode(
                          RISCV::ADDI, DL, VT, Addr.getOperand(0),
-                         CurDAG->getTargetConstant(AdjustedOffset, DL, VT)),
+                         CurDAG->getSignedConstant(AdjustedOffset, DL, VT,
+                                                   /*isTarget=*/true)),
                      0);
-      Offset = CurDAG->getTargetConstant(Adj, DL, VT);
+      Offset = CurDAG->getSignedConstant(Adj, DL, VT, /*isTarget=*/true);
       return true;
     }
 
@@ -2848,9 +2861,9 @@ bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
     if (Imm % ShiftWidth == ShiftWidth - 1) {
       SDLoc DL(ShAmt);
       EVT VT = ShAmt.getValueType();
-      MachineSDNode *Not =
-          CurDAG->getMachineNode(RISCV::XORI, DL, VT, ShAmt.getOperand(1),
-                                 CurDAG->getTargetConstant(-1, DL, VT));
+      MachineSDNode *Not = CurDAG->getMachineNode(
+          RISCV::XORI, DL, VT, ShAmt.getOperand(1),
+          CurDAG->getAllOnesConstant(DL, VT, /*isTarget=*/true));
       ShAmt = SDValue(Not, 0);
       return true;
     }
@@ -2897,21 +2910,21 @@ bool RISCVDAGToDAGISel::selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal,
     // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
     // non-zero otherwise.
     if (CVal == -2048) {
-      Val =
-          SDValue(CurDAG->getMachineNode(
-                      RISCV::XORI, DL, N->getValueType(0), LHS,
-                      CurDAG->getTargetConstant(CVal, DL, N->getValueType(0))),
-                  0);
+      Val = SDValue(CurDAG->getMachineNode(
+                        RISCV::XORI, DL, N->getValueType(0), LHS,
+                        CurDAG->getSignedConstant(CVal, DL, N->getValueType(0),
+                                                  /*isTarget=*/true)),
+                    0);
       return true;
     }
     // If the RHS is [-2047,2048], we can use addi with -RHS to produce 0 if the
     // LHS is equal to the RHS and non-zero otherwise.
     if (isInt<12>(CVal) || CVal == 2048) {
-      Val =
-          SDValue(CurDAG->getMachineNode(
-                      RISCV::ADDI, DL, N->getValueType(0), LHS,
-                      CurDAG->getTargetConstant(-CVal, DL, N->getValueType(0))),
-                  0);
+      Val = SDValue(CurDAG->getMachineNode(
+                        RISCV::ADDI, DL, N->getValueType(0), LHS,
+                        CurDAG->getSignedConstant(-CVal, DL, N->getValueType(0),
+                                                  /*isTarget=*/true)),
+                    0);
       return true;
     }
   }
@@ -3317,7 +3330,8 @@ bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
       return false;
 
     EVT Ty = N->getValueType(0);
-    Simm5 = CurDAG->getTargetConstant(Offset >> Shift, SDLoc(N), Ty);
+    Simm5 = CurDAG->getSignedConstant(Offset >> Shift, SDLoc(N), Ty,
+                                      /*isTarget=*/true);
     Shl2 = CurDAG->getTargetConstant(Shift, SDLoc(N), Ty);
     return true;
   }
@@ -3334,16 +3348,16 @@ bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
                                    N->getValueType(0));
   } else if (C && C->isAllOnes()) {
     // Treat all ones as VLMax.
-    VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N),
-                                   N->getValueType(0));
+    VL = CurDAG->getSignedConstant(RISCV::VLMaxSentinel, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
   } else if (isa<RegisterSDNode>(N) &&
              cast<RegisterSDNode>(N)->getReg() == RISCV::X0) {
     // All our VL operands use an operand that allows GPRNoX0 or an immediate
     // as the register class. Convert X0 to a special immediate to pass the
     // MachineVerifier. This is recognized specially by the vsetvli insertion
     // pass.
-    VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N),
-                                   N->getValueType(0));
+    VL = CurDAG->getSignedConstant(RISCV::VLMaxSentinel, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
   } else {
     VL = N;
   }
@@ -3401,7 +3415,8 @@ static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
   if (!ValidateImm(SplatImm))
     return false;
 
-  SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), Subtarget.getXLenVT());
+  SplatVal = DAG.getSignedConstant(SplatImm, SDLoc(N), Subtarget.getXLenVT(),
+                                   /*isTarget=*/true);
   return true;
 }
 
@@ -3495,7 +3510,8 @@ bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
     if (!isInt<5>(ImmVal))
       return false;
 
-    Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
+    Imm = CurDAG->getSignedConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT(),
+                                    /*isTarget=*/true);
     return true;
   }
 
@@ -3699,32 +3715,6 @@ static bool IsVMerge(SDNode *N) {
   return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMERGE_VVM;
 }
 
-static bool IsVMv(SDNode *N) {
-  return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMV_V_V;
-}
-
-static unsigned GetVMSetForLMul(RISCVII::VLMUL LMUL) {
-  switch (LMUL) {
-  case RISCVII::LMUL_F8:
-    return RISCV::PseudoVMSET_M_B1;
-  case RISCVII::LMUL_F4:
-    return RISCV::PseudoVMSET_M_B2;
-  case RISCVII::LMUL_F2:
-    return RISCV::PseudoVMSET_M_B4;
-  case RISCVII::LMUL_1:
-    return RISCV::PseudoVMSET_M_B8;
-  case RISCVII::LMUL_2:
-    return RISCV::PseudoVMSET_M_B16;
-  case RISCVII::LMUL_4:
-    return RISCV::PseudoVMSET_M_B32;
-  case RISCVII::LMUL_8:
-    return RISCV::PseudoVMSET_M_B64;
-  case RISCVII::LMUL_RESERVED:
-    llvm_unreachable("Unexpected LMUL");
-  }
-  llvm_unreachable("Unknown VLMUL enum");
-}
-
 // Try to fold away VMERGE_VVM instructions into their true operands:
 //
 // %true = PseudoVADD_VV ...
@@ -3739,35 +3729,22 @@ static unsigned GetVMSetForLMul(RISCVII::VLMUL LMUL) {
 // If %true is masked, then we can use its mask instead of vmerge's if vmerge's
 // mask is all ones.
 //
-// We can also fold a VMV_V_V into its true operand, since it is equivalent to a
-// VMERGE_VVM with an all ones mask.
-//
 // The resulting VL is the minimum of the two VLs.
 //
 // The resulting policy is the effective policy the vmerge would have had,
 // i.e. whether or not it's passthru operand was implicit-def.
 bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
   SDValue Passthru, False, True, VL, Mask, Glue;
-  // A vmv.v.v is equivalent to a vmerge with an all-ones mask.
-  if (IsVMv(N)) {
-    Passthru = N->getOperand(0);
-    False = N->getOperand(0);
-    True = N->getOperand(1);
-    VL = N->getOperand(2);
-    // A vmv.v.v won't have a Mask or Glue, instead we'll construct an all-ones
-    // mask later below.
-  } else {
-    assert(IsVMerge(N));
-    Passthru = N->getOperand(0);
-    False = N->getOperand(1);
-    True = N->getOperand(2);
-    Mask = N->getOperand(3);
-    VL = N->getOperand(4);
-    // We always have a glue node for the mask at v0.
-    Glue = N->getOperand(N->getNumOperands() - 1);
-  }
-  assert(!Mask || cast<RegisterSDNode>(Mask)->getReg() == RISCV::V0);
-  assert(!Glue || Glue.getValueType() == MVT::Glue);
+  assert(IsVMerge(N));
+  Passthru = N->getOperand(0);
+  False = N->getOperand(1);
+  True = N->getOperand(2);
+  Mask = N->getOperand(3);
+  VL = N->getOperand(4);
+  // We always have a glue node for the mask at v0.
+  Glue = N->getOperand(N->getNumOperands() - 1);
+  assert(cast<RegisterSDNode>(Mask)->getReg() == RISCV::V0);
+  assert(Glue.getValueType() == MVT::Glue);
 
   // If the EEW of True is different from vmerge's SEW, then we can't fold.
   if (True.getSimpleValueType() != N->getSimpleValueType(0))
@@ -3815,7 +3792,7 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
 
   // If True is masked then the vmerge must have either the same mask or an all
   // 1s mask, since we're going to keep the mask from True.
-  if (IsMasked && Mask) {
+  if (IsMasked) {
     // FIXME: Support mask agnostic True instruction which would have an
     // undef passthru operand.
     SDValue TrueMask =
@@ -3845,11 +3822,9 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
     SmallVector<const SDNode *, 4> LoopWorklist;
     SmallPtrSet<const SDNode *, 16> Visited;
     LoopWorklist.push_back(False.getNode());
-    if (Mask)
-      LoopWorklist.push_back(Mask.getNode());
+    LoopWorklist.push_back(Mask.getNode());
     LoopWorklist.push_back(VL.getNode());
-    if (Glue)
-      LoopWorklist.push_back(Glue.getNode());
+    LoopWorklist.push_back(Glue.getNode());
     if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
       return false;
   }
@@ -3910,21 +3885,6 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
     Glue = True->getOperand(True->getNumOperands() - 1);
     assert(Glue.getValueType() == MVT::Glue);
   }
-  // If we end up using the vmerge mask the vmerge is actually a vmv.v.v, create
-  // an all-ones mask to use.
-  else if (IsVMv(N)) {
-    unsigned TSFlags = TII->get(N->getMachineOpcode()).TSFlags;
-    unsigned VMSetOpc = GetVMSetForLMul(RISCVII::getLMul(TSFlags));
-    ElementCount EC = N->getValueType(0).getVectorElementCount();
-    MVT MaskVT = MVT::getVectorVT(MVT::i1, EC);
-
-    SDValue AllOnesMask =
-        SDValue(CurDAG->getMachineNode(VMSetOpc, DL, MaskVT, VL, SEW), 0);
-    SDValue MaskCopy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
-                                            RISCV::V0, AllOnesMask, SDValue());
-    Mask = CurDAG->getRegister(RISCV::V0, MaskVT);
-    Glue = MaskCopy.getValue(1);
-  }
 
   unsigned MaskedOpc = Info->MaskedPseudo;
 #ifndef NDEBUG
@@ -4003,7 +3963,7 @@ bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
     if (N->use_empty() || !N->isMachineOpcode())
       continue;
 
-    if (IsVMerge(N) || IsVMv(N))
+    if (IsVMerge(N))
       MadeChange |= performCombineVMergeAndVOps(N);
   }
   return MadeChange;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 911fa45d7173e88..62980ae71d60b69 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -784,9 +784,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
       // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
       // nodes which truncate by one power of two at a time.
-      setOperationAction({ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S,
-                          ISD::TRUNCATE_SSAT_U, ISD::TRUNCATE_USAT_U},
-                         VT, Custom);
+      setOperationAction(
+          {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
+          Custom);
 
       // Custom-lower insert/extract operations to simplify patterns.
       setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
@@ -1087,8 +1087,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
                            Custom);
 
-        setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS}, VT,
-                           Custom);
+        setOperationAction(
+            {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
+            Custom);
 
         setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
                            VT, Custom);
@@ -1101,9 +1102,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
         setOperationAction(ISD::SELECT, VT, Custom);
 
-        setOperationAction({ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S,
-                            ISD::TRUNCATE_SSAT_U, ISD::TRUNCATE_USAT_U},
-                           VT, Custom);
+        setOperationAction(
+            {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
+            Custom);
 
         setOperationAction(ISD::BITCAST, VT, Custom);
 
@@ -1235,8 +1236,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         // expansion to a build_vector of 0s.
         setOperationAction(ISD::UNDEF, VT, Custom);
 
-        setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
-                            ISD::EXTRACT_SUBVECTOR},
+        setOperationAction({ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE,
+                            ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR},
                            VT, Custom);
 
         // FIXME: mload, mstore, mgather, mscatter, vp_load/store,
@@ -3595,7 +3596,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
       if (I % NumViaIntegerBits == 0 || I == NumElts) {
         if (NumViaIntegerBits <= 32)
           Bits = SignExtend64<32>(Bits);
-        SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
+        SDValue Elt = DAG.getSignedConstant(Bits, DL, XLenVT);
         Elts[IntegerEltIdx] = Elt;
         Bits = 0;
         BitPos = 0;
@@ -3674,7 +3675,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
       VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
       if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
           (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
-        SDValue SplatStep = DAG.getConstant(SplatStepVal, DL, VIDVT);
+        SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
         VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
       }
       if (StepDenominator != 1) {
@@ -3683,7 +3684,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
         VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
       }
       if (Addend != 0 || Negate) {
-        SDValue SplatAddend = DAG.getConstant(Addend, DL, VIDVT);
+        SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
         VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
                           VID);
       }
@@ -3722,12 +3723,13 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
 
     // On RV64, sign-extend from 32 to 64 bits where possible in order to
     // achieve better constant materializion.
-    if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
+    // On RV32, we need to sign-extend to use getSignedConstant.
+    if (ViaIntVT == MVT::i32)
       SplatValue = SignExtend64<32>(SplatValue);
 
     SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ViaVecVT,
                               DAG.getUNDEF(ViaVecVT),
-                              DAG.getConstant(SplatValue, DL, XLenVT),
+                              DAG.getSignedConstant(SplatValue, DL, XLenVT),
                               DAG.getVectorIdxConstant(0, DL));
     if (ViaVecLen != 1)
       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
@@ -3779,7 +3781,8 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
 
     // On RV64, sign-extend from 32 to 64 bits where possible in order to
     // achieve better constant materializion.
-    if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
+    // On RV32, we need to sign-extend to use getSignedConstant.
+    if (ViaIntVT == MVT::i32)
       SplatValue = SignExtend64<32>(SplatValue);
 
     // Since we can't introduce illegal i64 types at this stage, we can only
@@ -3796,7 +3799,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
       SDValue Splat =
           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
                       DAG.getUNDEF(ViaContainerVT),
-                      DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
+                      DAG.getSignedConstant(SplatValue, DL, XLenVT), ViaVL);
       Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
       if (ViaVecLen != RequiredVL)
         Splat = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
@@ -3813,16 +3816,14 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
   // TODO: We really should be costing the smaller vector.  There are
   // profitable cases this misses.
   if (EltBitSize > 8 && VT.isInteger() &&
-      (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen())) {
-    unsigned SignBits = DAG.ComputeNumSignBits(Op);
-    if (EltBitSize - SignBits < 8) {
-      SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
-                                          DL, Op->ops());
-      Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
-                                       Source, DAG, Subtarget);
-      SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
-      return convertFromScalableVector(VT, Res, DAG, Subtarget);
-    }
+      (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
+      DAG.ComputeMaxSignificantBits(Op) <= 8) {
+    SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
+                                        DL, Op->ops());
+    Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
+                                     Source, DAG, Subtarget);
+    SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
+    return convertFromScalableVector(VT, Res, DAG, Subtarget);
   }
 
   if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
@@ -5162,6 +5163,9 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     return convertFromScalableVector(VT, Res, DAG, Subtarget);
   }
 
+  if (ShuffleVectorInst::isReverseMask(Mask, NumElts) && V2.isUndef())
+    return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
+
   // If this is a deinterleave and we can widen the vector, then we can use
   // vnsrl to deinterleave.
   if (isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget)) {
@@ -6230,7 +6234,6 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   }
   case ISD::TRUNCATE:
   case ISD::TRUNCATE_SSAT_S:
-  case ISD::TRUNCATE_SSAT_U:
   case ISD::TRUNCATE_USAT_U:
     // Only custom-lower vector truncates
     if (!Op.getSimpleValueType().isVector())
@@ -6809,7 +6812,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
           // Using getSetCCSwappedOperands will convert SET(U)GT->SET(U)LT.
           CCVal = ISD::getSetCCSwappedOperands(CCVal);
           SDValue SetCC = DAG.getSetCC(
-              DL, VT, LHS, DAG.getConstant(Imm + 1, DL, OpVT), CCVal);
+              DL, VT, LHS, DAG.getSignedConstant(Imm + 1, DL, OpVT), CCVal);
           return DAG.getLogicalNOT(DL, SetCC, VT);
         }
       }
@@ -7759,8 +7762,9 @@ SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
   unsigned Depth = Op.getConstantOperandVal(0);
   while (Depth--) {
     int Offset = -(XLenInBytes * 2);
-    SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
-                              DAG.getIntPtrConstant(Offset, DL));
+    SDValue Ptr = DAG.getNode(
+        ISD::ADD, DL, VT, FrameAddr,
+        DAG.getSignedConstant(Offset, DL, getPointerTy(DAG.getDataLayout())));
     FrameAddr =
         DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
   }
@@ -7785,7 +7789,7 @@ SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
   if (Depth) {
     int Off = -XLenInBytes;
     SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
-    SDValue Offset = DAG.getConstant(Off, DL, VT);
+    SDValue Offset = DAG.getSignedConstant(Off, DL, VT);
     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
                        MachinePointerInfo());
@@ -7814,7 +7818,7 @@ SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
 
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue One = DAG.getConstant(1, DL, VT);
-  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
+  SDValue MinusXLen = DAG.getSignedConstant(-(int)Subtarget.getXLen(), DL, VT);
   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
@@ -7864,7 +7868,7 @@ SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
 
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue One = DAG.getConstant(1, DL, VT);
-  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
+  SDValue MinusXLen = DAG.getSignedConstant(-(int)Subtarget.getXLen(), DL, VT);
   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
@@ -7957,7 +7961,7 @@ SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
 
   if (VecVT.isScalableVector()) {
     SDValue SplatZero = DAG.getConstant(0, DL, VecVT);
-    SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, VecVT);
+    SDValue SplatTrueVal = DAG.getSignedConstant(ExtTrueVal, DL, VecVT);
     return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
   }
 
@@ -7971,7 +7975,7 @@ SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
 
   MVT XLenVT = Subtarget.getXLenVT();
   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
-  SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
+  SDValue SplatTrueVal = DAG.getSignedConstant(ExtTrueVal, DL, XLenVT);
 
   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                           DAG.getUNDEF(ContainerVT), SplatZero, VL);
@@ -8117,7 +8121,7 @@ SDValue RISCVTargetLowering::lowerVectorTruncLike(SDValue Op,
   unsigned NewOpc;
   if (Opc == ISD::TRUNCATE_SSAT_S)
     NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
-  else if (Opc == ISD::TRUNCATE_SSAT_U || Opc == ISD::TRUNCATE_USAT_U)
+  else if (Opc == ISD::TRUNCATE_USAT_U)
     NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
   else
     NewOpc = RISCVISD::TRUNCATE_VECTOR_VL;
@@ -8358,7 +8362,7 @@ SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
     const auto *CVal = cast<ConstantSDNode>(Val);
     if (isInt<32>(CVal->getSExtValue())) {
       IsLegalInsert = true;
-      Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
+      Val = DAG.getSignedConstant(CVal->getSExtValue(), DL, MVT::i32);
     }
   }
 
@@ -10310,19 +10314,27 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
     SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1);
     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Op2);
   }
-  unsigned EltSize = VecVT.getScalarSizeInBits();
-  unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
+
+  MVT ContainerVT = VecVT;
+  SDValue Vec = Op.getOperand(0);
+  if (VecVT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(VecVT);
+    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
+  }
+
+  unsigned EltSize = ContainerVT.getScalarSizeInBits();
+  unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
   unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
   unsigned MaxVLMAX =
-    RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
+      VecVT.isFixedLengthVector()
+          ? VecVT.getVectorNumElements()
+          : RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
 
   unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
-  MVT IntVT = VecVT.changeVectorElementTypeToInteger();
+  MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
 
   // If this is SEW=8 and VLMAX is potentially more than 256, we need
   // to use vrgatherei16.vv.
-  // TODO: It's also possible to use vrgatherei16.vv for other types to
-  // decrease register width for the index calculation.
   if (MaxVLMAX > 256 && EltSize == 8) {
     // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
     // Reverse each half, then reassemble them in reverse order.
@@ -10344,17 +10356,27 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
     }
 
     // Just promote the int type to i16 which will double the LMUL.
-    IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
+    IntVT = MVT::getVectorVT(MVT::i16, ContainerVT.getVectorElementCount());
+    GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
+  }
+
+  // At LMUL > 1, do the index computation in 16 bits to reduce register
+  // pressure.
+  if (IntVT.getScalarType().bitsGT(MVT::i16) &&
+      IntVT.bitsGT(getLMUL1VT(IntVT))) {
+    assert(isUInt<16>(MaxVLMAX - 1)); // Largest VLMAX is 65536 @ zvl65536b
     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
+    IntVT = IntVT.changeVectorElementType(MVT::i16);
   }
 
   MVT XLenVT = Subtarget.getXLenVT();
-  auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
+  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
   // Calculate VLMAX-1 for the desired SEW.
-  SDValue VLMinus1 = DAG.getNode(ISD::SUB, DL, XLenVT,
-                                 computeVLMax(VecVT, DL, DAG),
-                                 DAG.getConstant(1, DL, XLenVT));
+  SDValue VLMinus1 = DAG.getNode(
+      ISD::SUB, DL, XLenVT,
+      DAG.getElementCount(DL, XLenVT, VecVT.getVectorElementCount()),
+      DAG.getConstant(1, DL, XLenVT));
 
   // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
   bool IsRV32E64 =
@@ -10370,8 +10392,11 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
   SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
                                 DAG.getUNDEF(IntVT), Mask, VL);
 
-  return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices,
-                     DAG.getUNDEF(VecVT), Mask, VL);
+  SDValue Gather = DAG.getNode(GatherOpc, DL, ContainerVT, Vec, Indices,
+                               DAG.getUNDEF(ContainerVT), Mask, VL);
+  if (VecVT.isFixedLengthVector())
+    Gather = convertFromScalableVector(VecVT, Gather, DAG, Subtarget);
+  return Gather;
 }
 
 SDValue RISCVTargetLowering::lowerVECTOR_SPLICE(SDValue Op,
@@ -10964,7 +10989,7 @@ SDValue RISCVTargetLowering::lowerVPExtMaskOp(SDValue Op,
   SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                                   DAG.getUNDEF(ContainerVT), Zero, VL);
 
-  SDValue SplatValue = DAG.getConstant(
+  SDValue SplatValue = DAG.getSignedConstant(
       Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, XLenVT);
   SDValue Splat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                               DAG.getUNDEF(ContainerVT), SplatValue, VL);
@@ -11094,7 +11119,7 @@ SDValue RISCVTargetLowering::lowerVPFPIntConvOp(SDValue Op,
         SDValue Zero = DAG.getConstant(0, DL, XLenVT);
         SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
                                         DAG.getUNDEF(IntVT), Zero, VL);
-        SDValue One = DAG.getConstant(
+        SDValue One = DAG.getSignedConstant(
             RISCVISDExtOpc == RISCVISD::VZEXT_VL ? 1 : -1, DL, XLenVT);
         SDValue OneSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
                                        DAG.getUNDEF(IntVT), One, VL);
@@ -12976,10 +13001,10 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
   // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
   SDLoc DL(N);
   SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
-                             DAG.getConstant(CA, DL, VT));
+                             DAG.getSignedConstant(CA, DL, VT));
   SDValue New1 =
-      DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
-  return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
+      DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getSignedConstant(C0, DL, VT));
+  return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getSignedConstant(CB, DL, VT));
 }
 
 // add (zext, zext) -> zext (add (zext, zext))
@@ -15863,10 +15888,11 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   auto [StrideVariant, MustNegateStride] = *BaseDiff;
-  SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
-                       ? std::get<SDValue>(StrideVariant)
-                       : DAG.getConstant(std::get<int64_t>(StrideVariant), DL,
-                                         Lds[0]->getOffset().getValueType());
+  SDValue Stride =
+      std::holds_alternative<SDValue>(StrideVariant)
+          ? std::get<SDValue>(StrideVariant)
+          : DAG.getSignedConstant(std::get<int64_t>(StrideVariant), DL,
+                                  Lds[0]->getOffset().getValueType());
   if (MustNegateStride)
     Stride = DAG.getNegative(Stride, DL, Stride.getValueType());
 
@@ -16577,7 +16603,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                         DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
         SDValue AND =
             DAG.getNode(ISD::AND, DL, VT, SRA,
-                        DAG.getConstant(TrueSImm - FalseSImm, DL, VT));
+                        DAG.getSignedConstant(TrueSImm - FalseSImm, DL, VT));
         return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV);
       }
 
@@ -16732,14 +16758,14 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
         assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
         assert(IndexType == ISD::UNSIGNED_SCALED);
         SDValue BasePtr = DAG.getNode(ISD::ADD, DL, PtrVT, MGN->getBasePtr(),
-                                      DAG.getConstant(Addend, DL, PtrVT));
+                                      DAG.getSignedConstant(Addend, DL, PtrVT));
 
         SDValue EVL = DAG.getElementCount(DL, Subtarget.getXLenVT(),
                                           VT.getVectorElementCount());
-        SDValue StridedLoad =
-            DAG.getStridedLoadVP(VT, DL, MGN->getChain(), BasePtr,
-                                 DAG.getConstant(StepNumerator, DL, XLenVT),
-                                 MGN->getMask(), EVL, MGN->getMemOperand());
+        SDValue StridedLoad = DAG.getStridedLoadVP(
+            VT, DL, MGN->getChain(), BasePtr,
+            DAG.getSignedConstant(StepNumerator, DL, XLenVT), MGN->getMask(),
+            EVL, MGN->getMemOperand());
         SDValue VPSelect = DAG.getNode(ISD::VP_SELECT, DL, VT, MGN->getMask(),
                                        StridedLoad, MGN->getPassThru(), EVL);
         return DAG.getMergeValues({VPSelect, SDValue(StridedLoad.getNode(), 1)},
@@ -17195,7 +17221,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
       EVT VT = N->getValueType(0);
       if (IntNo == Intrinsic::riscv_vfirst ||
           IntNo == Intrinsic::riscv_vfirst_mask)
-        return DAG.getConstant(-1, DL, VT);
+        return DAG.getAllOnesConstant(DL, VT);
       return DAG.getConstant(0, DL, VT);
     }
     }
@@ -20476,8 +20502,8 @@ void RISCVTargetLowering::LowerAsmOperandForConstraint(
       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
         uint64_t CVal = C->getSExtValue();
         if (isInt<12>(CVal))
-          Ops.push_back(
-              DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
+          Ops.push_back(DAG.getSignedConstant(
+              CVal, SDLoc(Op), Subtarget.getXLenVT(), /*isTarget=*/true));
       }
       return;
     case 'J':
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 2dd7240ef941a73..74406bf4b104719 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -403,14 +403,14 @@ def AddrRegImm : ComplexPattern<iPTR, 2, "SelectAddrRegImm">;
 
 // Return the negation of an immediate value.
 def NegImm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(-N->getSExtValue(), SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(-N->getSExtValue(), SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 // Return an immediate value minus 32.
 def ImmSub32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue() - 32, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(N->getSExtValue() - 32, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 // Return an immediate subtracted from XLen.
@@ -442,16 +442,16 @@ def AddiPair : PatLeaf<(imm), [{
 def AddiPairImmSmall : SDNodeXForm<imm, [{
   int64_t Imm = N->getSExtValue();
   int64_t Adj = N->getSExtValue() < 0 ? -2048 : 2047;
-  return CurDAG->getTargetConstant(Imm - Adj, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(Imm - Adj, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 // Return -2048 if immediate is negative or 2047 if positive. These are the
 // largest simm12 values.
 def AddiPairImmLarge : SDNodeXForm<imm, [{
   int64_t Imm = N->getSExtValue() < 0 ? -2048 : 2047;
-  return CurDAG->getTargetConstant(Imm, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(Imm, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 def TrailingZeros : SDNodeXForm<imm, [{
@@ -1056,7 +1056,7 @@ def : InstAlias<"sltu $rd, $rs1, $imm12",
 
 def : MnemonicAlias<"move", "mv">;
 
-// The SCALL and SBREAK instructions wererenamed to ECALL and EBREAK in
+// The SCALL and SBREAK instructions were renamed to ECALL and EBREAK in
 // version 2.1 of the user-level ISA. Like the GNU toolchain, we still accept
 // the old name for backwards compatibility.
 def : MnemonicAlias<"scall", "ecall">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index ab4d5a7e180df00..1b4303fbbcf8093 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -92,8 +92,8 @@ def AVL : RegisterOperand<GPRNoX0> {
 def VLOp : ComplexPattern<XLenVT, 1, "selectVLOp">;
 
 def DecImm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue() - 1, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(N->getSExtValue() - 1, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 defvar TAIL_AGNOSTIC = 1;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 8003bd98b8706a6..8e6f281027695a2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -45,30 +45,6 @@ def riscv_clmul   : SDNode<"RISCVISD::CLMUL",   SDTIntBinOp>;
 def riscv_clmulh  : SDNode<"RISCVISD::CLMULH",  SDTIntBinOp>;
 def riscv_clmulr  : SDNode<"RISCVISD::CLMULR",  SDTIntBinOp>;
 
-def UImmLog2XLenHalfAsmOperand : AsmOperandClass {
-  let Name = "UImmLog2XLenHalf";
-  let RenderMethod = "addImmOperands";
-  let DiagnosticType = "InvalidUImmLog2XLenHalf";
-}
-
-def shfl_uimm : RISCVOp, ImmLeaf<XLenVT, [{
-  if (Subtarget->is64Bit())
-    return isUInt<5>(Imm);
-  return isUInt<4>(Imm);
-}]> {
-  let ParserMatchClass = UImmLog2XLenHalfAsmOperand;
-  let DecoderMethod = "decodeUImmOperand<5>";
-  let OperandType = "OPERAND_UIMM_SHFL";
-  let MCOperandPredicate = [{
-    int64_t Imm;
-    if (!MCOp.evaluateAsConstantImm(Imm))
-      return false;
-    if (STI.getTargetTriple().isArch64Bit())
-      return  isUInt<5>(Imm);
-    return isUInt<4>(Imm);
-  }];
-}
-
 def BCLRXForm : SDNodeXForm<imm, [{
   // Find the lowest 0.
   return CurDAG->getTargetConstant(llvm::countr_one(N->getZExtValue()),
@@ -148,9 +124,9 @@ def BCLRITwoBitsMaskLow : SDNodeXForm<imm, [{
 }]>;
 
 def BCLRITwoBitsMaskHigh : SDNodeXForm<imm, [{
-  uint64_t I = N->getSExtValue();
+  uint64_t I = N->getZExtValue();
   if (!Subtarget->is64Bit())
-    I |= 0xffffffffull << 32;
+    I |= maskLeadingOnes<uint64_t>(32);
   return CurDAG->getTargetConstant(llvm::Log2_64(~I), SDLoc(N),
                                    N->getValueType(0));
 }]>;
@@ -169,8 +145,8 @@ def BCLRIANDIMask : PatLeaf<(imm), [{
 }]>;
 
 def BCLRIANDIMaskLow : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant((N->getZExtValue() & 0x7ff) | ~0x7ffull,
-                                   SDLoc(N), N->getValueType(0));
+  return CurDAG->getSignedConstant((N->getZExtValue() & 0x7ff) | ~0x7ffull,
+                                   SDLoc(N), N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 def CSImm12MulBy4 : PatLeaf<(imm), [{
@@ -191,13 +167,13 @@ def CSImm12MulBy8 : PatLeaf<(imm), [{
 }]>;
 
 def SimmShiftRightBy2XForm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue() >> 2, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(N->getSExtValue() >> 2, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 def SimmShiftRightBy3XForm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue() >> 3, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(N->getSExtValue() >> 3, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 // Pattern to exclude simm12 immediates from matching, namely `non_imm12`.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index bf9caa9b30b674e..cafd259031746d5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -127,10 +127,10 @@ let Predicates = [HasStdExtZvbb] in {
   defm VWSLL_V  : VSHT_IV_V_X_I<"vwsll", 0b110101>;
 } // Predicates = [HasStdExtZvbb]
 
-let Predicates = [HasStdExtZvbc] in {
+let Predicates = [HasStdExtZvbcOrZvbc32e] in {
   defm VCLMUL_V  : VCLMUL_MV_V_X<"vclmul", 0b001100>;
   defm VCLMULH_V : VCLMUL_MV_V_X<"vclmulh", 0b001101>;
-} // Predicates = [HasStdExtZvbc]
+} // Predicates = [HasStdExtZvbcOrZvbc32e]
 
 let Predicates = [HasStdExtZvkb] in {
   defm VANDN_V  : VALU_IV_V_X<"vandn", 0b000001>;
@@ -150,6 +150,14 @@ let Predicates = [HasStdExtZvkg], RVVConstraint = NoConstraint in {
                  SchedBinaryMC<"WriteVGMULV", "ReadVGMULV", "ReadVGMULV">;
 } // Predicates = [HasStdExtZvkg]
 
+let Predicates = [HasStdExtZvkgs], RVVConstraint = VS2Constraint in {
+  def VGHSH_VS : PALUVVNoVmTernary<0b100011, OPMVV, "vghsh.vs">,
+                 SchedTernaryMC<"WriteVGHSHV", "ReadVGHSHV", "ReadVGHSHV",
+                                "ReadVGHSHV">;
+  def VGMUL_VS : PALUVs2NoVmBinary<0b101001, 0b10001, OPMVV, "vgmul.vs">,
+                 SchedBinaryMC<"WriteVGMULV", "ReadVGMULV", "ReadVGMULV">;
+} // Predicates = [HasStdExtZvkgs]
+
 let Predicates = [HasStdExtZvknhaOrZvknhb], RVVConstraint = Sha2Constraint in {
   def VSHA2CH_VV : PALUVVNoVmTernary<0b101110, OPMVV, "vsha2ch.vv">,
                    SchedTernaryMC<"WriteVSHA2CHV", "ReadVSHA2CHV", "ReadVSHA2CHV",
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
index 9fa455be3852577..6926184e92399c2 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
@@ -8,6 +8,115 @@
 
 //===----------------------------------------------------------------------===//
 
+/// c is true if mx has the worst case behavior compared to LMULs in MxList.
+/// On the SiFiveP400, the worst case LMUL is the Largest LMUL
+/// and the worst case sew is the smallest SEW for that LMUL.
+class SiFiveP400IsWorstCaseMX<string mx, list<string> MxList> {
+  string LLMUL = LargestLMUL<MxList>.r;
+  bit c = !eq(mx, LLMUL);
+}
+
+class SiFiveP400IsWorstCaseMXSEW<string mx, int sew, list<string> MxList, bit isF = 0> {
+  string LLMUL = LargestLMUL<MxList>.r;
+  int SSEW = SmallestSEW<mx, isF>.r;
+  bit c = !and(!eq(mx, LLMUL), !eq(sew, SSEW));
+}
+
+// 1 Micro-Op per cycle.
+class SiFiveP400GetLMulCycles<string mx> {
+  int c = !cond(
+    !eq(mx, "M1") : 1,
+    !eq(mx, "M2") : 2,
+    !eq(mx, "M4") : 4,
+    !eq(mx, "M8") : 8,
+    !eq(mx, "MF2") : 1,
+    !eq(mx, "MF4") : 1,
+    !eq(mx, "MF8") : 1
+  );
+}
+
+// Latency for segmented loads and stores are calculated as vl * nf.
+class SiFiveP400GetCyclesSegmented<string mx, int sew, int nf> {
+  defvar VLEN = 128;
+  defvar VLUpperBound = !cond(
+    !eq(mx, "M1") : !div(VLEN, sew),
+    !eq(mx, "M2") : !div(!mul(VLEN, 2), sew),
+    !eq(mx, "M4") : !div(!mul(VLEN, 4), sew),
+    !eq(mx, "M8") : !div(!mul(VLEN, 8), sew),
+    !eq(mx, "MF2") : !div(!div(VLEN, 2), sew),
+    !eq(mx, "MF4") : !div(!div(VLEN, 4), sew),
+    !eq(mx, "MF8") : !div(!div(VLEN, 8), sew),
+  );
+  int c = !mul(VLUpperBound, nf);
+}
+
+// Both variants of floating point vector reductions are based on numbers collected
+// from llvm-exegesis.
+class VFReduceBaseCycles<int sew> {
+  // The latency for simple unordered VFReduce is `C + 6 * log2(LMUL)`,
+  // and `C * LMUL` for ordered VFReduce. This helper class provides the `C`.
+  int val = !cond(!eq(sew, 16): 16,
+                  !eq(sew, 32): 10,
+                  !eq(sew, 64): 6);
+}
+
+class AdvancedVFReduceCycles<int sew, string mx> {
+  // SEW = 64 has lower latencies and RThroughputs than other SEWs.
+  int latency = !cond(!eq(mx, "M1"): !if(!eq(sew, 64), 4, 6),
+                      !eq(mx, "M2"): !if(!eq(sew, 64), 6, 8),
+                      !eq(mx, "M4"): !if(!eq(sew, 64), 8, 10),
+                      !eq(mx, "M8"): !if(!eq(sew, 64), 11, 13),
+                      true: !if(!eq(sew, 64), 4, 6));
+  int rthroughput = !cond(!eq(mx, "M1"): !if(!eq(sew, 64), 2, 3),
+                          !eq(mx, "M2"): !if(!eq(sew, 64), 3, 4),
+                          !eq(mx, "M4"): !if(!eq(sew, 64), 5, 6),
+                          !eq(mx, "M8"): !if(!eq(sew, 64), 10, 12),
+                          true: !if(!eq(sew, 64), 2, 3));
+}
+
+// Both variants of integer vector reductions are based on numbers collected
+// from llvm-exegesis.
+// TODO: Fractional LMUL's latency and rthroughput.
+class SimpleVIReduceCycles<string mx> {
+  defvar LMul = SiFiveP400GetLMulCycles<mx>.c;
+  int latency = !mul(LMul, 2);
+  int rthroughput = !cond(
+                      !eq(mx, "M1"): 1,
+                      !eq(mx, "M2"): 2,
+                      !eq(mx, "M4"): 4,
+                      !eq(mx, "M8"): 9,
+                      true: 1);
+}
+
+class AdvancedVIReduceCycles<int sew, string mx> {
+  // `C - 2 * log2(SEW)`, where `C` = 16.1, 18.1, 20.1, and 23.8 for
+  // M1/2/4/8, respectively.
+  int latency = !cond(!eq(mx, "M1"): !sub(16, !mul(2, !logtwo(sew))),
+                      !eq(mx, "M2"): !sub(18, !mul(2, !logtwo(sew))),
+                      !eq(mx, "M4"): !sub(20, !mul(2, !logtwo(sew))),
+                      !eq(mx, "M8"): !sub(23, !mul(2, !logtwo(sew))),
+                      true: 4);
+  int rthroughput = !cond(
+                      // `8.3 - 1.02 * log2(SEW)`
+                      !eq(mx, "M1"): !sub(8, !logtwo(sew)),
+                      // `10.0 - 1.16 * log2(SEW)`. Note that `9 - log2(SEW)`
+                      // is closer to the floor value of the original formula.
+                      !eq(mx, "M2"): !sub(9, !logtwo(sew)),
+                      // `14.2 - 1.53 * log2(SEW)`
+                      !eq(mx, "M4"): !div(!sub(1420, !mul(153, !logtwo(sew))), 100),
+                      // `24.1 - 2.3 * log2(SEW)`
+                      !eq(mx, "M8"): !div(!sub(241, !mul(23, !logtwo(sew))), 10),
+                      true: 1);
+}
+
+class SiFiveP400VSM3CCycles<string mx> {
+  // c = ceil(LMUL / 2)
+  int c = !cond(!eq(mx, "M2") : 1,
+                !eq(mx, "M4") : 2,
+                !eq(mx, "M8") : 4,
+                true : 1);
+}
+
 def SiFiveP400Model : SchedMachineModel {
   let IssueWidth = 3;         // 3 micro-ops are dispatched per cycle.
   let MicroOpBufferSize = 56; // Max micro-ops that can be buffered.
@@ -45,6 +154,13 @@ defvar SiFiveP400FloatArith  = SiFiveP400FEXQ0;
 defvar SiFiveP400F2I      = SiFiveP400FEXQ0;
 def SiFiveP400FloatDiv    : ProcResource<1>;
 
+// Vector pipeline
+def SiFiveP400VEXQ0        : ProcResource<1>;
+def SiFiveP400VLD          : ProcResource<1>;
+def SiFiveP400VST          : ProcResource<1>;
+def SiFiveP400VDiv         : ProcResource<1>;
+def SiFiveP400VFloatDiv    : ProcResource<1>;
+
 let Latency = 1 in {
 // Integer arithmetic and logic
 def : WriteRes<WriteIALU, [SiFiveP400IntArith]>;
@@ -246,9 +362,549 @@ def : WriteRes<WriteFMovI64ToF64, [SiFiveP400I2F]>;
 def : WriteRes<WriteFMovF64ToI64, [SiFiveP400F2I]>;
 }
 
+// 6. Configuration-Setting Instructions
+def : WriteRes<WriteVSETVLI, [SiFiveP400SYS]>;
+def : WriteRes<WriteVSETIVLI, [SiFiveP400SYS]>;
+def : WriteRes<WriteVSETVL, [SiFiveP400SYS]>;
+
+// 7. Vector Loads and Stores
+// FIXME: This unit is still being improved, currently
+// it is based on stage numbers. Estimates are optimistic,
+// latency may be longer.
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 8, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVLDE",    [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDM",    [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDFF",   [SiFiveP400VLD], mx, IsWorstCase>;
+  }
+  let Latency = 12, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVLDS8",   [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDS16",  [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDS32",  [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDS64",  [SiFiveP400VLD], mx, IsWorstCase>;
+  }
+  let Latency = 12, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVLDUX8",  [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDUX16", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDUX32", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDUX64", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDOX8",  [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDOX16", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDOX32", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDOX64", [SiFiveP400VLD], mx, IsWorstCase>;
+  }
+}
+
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 8, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSTE",    [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTM",    [SiFiveP400VST], mx, IsWorstCase>;
+  }
+  let Latency = 12, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSTS8",   [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTS16",  [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTS32",  [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTS64",  [SiFiveP400VST], mx, IsWorstCase>;
+  }
+  let Latency = 12, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSTUX8",  [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTUX16", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTUX32", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTUX64", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTOX8",  [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTOX16", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTOX32", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTOX64", [SiFiveP400VST], mx, IsWorstCase>;
+  }
+}
+
+foreach mx = SchedMxList in {
+  foreach nf=2-8 in {
+    foreach eew = [8, 16, 32, 64] in {
+      defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+      defvar LMulLat = SiFiveP400GetCyclesSegmented<mx, eew, nf>.c;
+      let Latency = !add(12, LMulLat), ReleaseAtCycles = [!add(12, LMulLat)] in {
+        defm "" : LMULWriteResMX<"WriteVLSEG" # nf # "e" #eew,   [SiFiveP400VLD], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVLSEGFF" # nf # "e" #eew, [SiFiveP400VLD], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVLSSEG" # nf # "e" #eew,  [SiFiveP400VLD], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVLUXSEG" # nf # "e" #eew, [SiFiveP400VLD], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVLOXSEG" # nf # "e" #eew, [SiFiveP400VLD], mx, IsWorstCase>;
+      }
+      let Latency = !add(1, LMulLat), ReleaseAtCycles = [!add(12, LMulLat)] in {
+        defm "" : LMULWriteResMX<"WriteVSSEG" # nf # "e" #eew,   [SiFiveP400VST], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVSSSEG" # nf # "e" #eew,  [SiFiveP400VST], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVSUXSEG" # nf # "e" #eew, [SiFiveP400VST], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVSOXSEG" # nf # "e" #eew, [SiFiveP400VST], mx, IsWorstCase>;
+      }
+    }
+  }
+}
+
+// Whole register move/load/store
+foreach LMul = [1, 2, 4, 8] in {
+  let Latency = 8, ReleaseAtCycles = [LMul] in {
+    def : WriteRes<!cast<SchedWrite>("WriteVLD" # LMul # "R"), [SiFiveP400VLD]>;
+    def : WriteRes<!cast<SchedWrite>("WriteVST" # LMul # "R"), [SiFiveP400VST]>;
+  }
+  let Latency = 2, ReleaseAtCycles = [LMul] in {
+    def : WriteRes<!cast<SchedWrite>("WriteVMov" # LMul # "V"), [SiFiveP400VEXQ0]>;
+  }
+}
+
+// 11. Vector Integer Arithmetic Instructions
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVIALUV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIALUX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIALUI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVExtV",     [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICmpV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICmpX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICmpI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMinMaxV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMinMaxX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMergeV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMergeX",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMergeI",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMovV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMovX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMovI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+
+  let Latency = !if(!lt(LMulLat, 2), 2, LMulLat), ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVShiftV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVShiftX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVShiftI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+
+  let Latency = !if(!eq(mx, "M8"), 9, 6), ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVIMulV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulAddV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulAddX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+// Widening
+foreach mx = SchedMxListW in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxListW>.c;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVIWALUV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWALUX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWALUI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWMulV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWMulX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWMulAddV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWMulAddX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// Worst case needs 51/45/42/72 * lmul cycles for i8/16/32/64.
+foreach mx = SchedMxList in {
+  foreach sew = SchedSEWSet<mx>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+    defvar DivMicroOpLat =
+      !cond(!eq(sew, 8): 51, !eq(sew, 16): 45, !eq(sew, 32): 42,
+      /* SEW=64 */ true: 72);
+    defvar DivLatency = !mul(DivMicroOpLat, LMulLat);
+    let Latency = DivLatency, ReleaseAtCycles = [LMulLat, DivLatency] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVIDivV", [SiFiveP400VEXQ0, SiFiveP400VDiv], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVIDivX", [SiFiveP400VEXQ0, SiFiveP400VDiv], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// Narrowing Shift and Clips
+foreach mx = SchedMxListW in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxListW>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVNShiftV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNShiftX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNShiftI", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNClipV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNClipX",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNClipI",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// 12. Vector Fixed-Point Arithmetic Instructions
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSALUV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSALUX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSALUI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAALUV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAALUX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSMulV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSMulX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSShiftV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSShiftX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSShiftI", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// 13. Vector Floating-Point Instructions
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+    let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFALUV",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulV",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+    let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFCvtIToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+  }
+}
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVFCmpV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFCmpF",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFClassV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFMergeV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFMovV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVFCvtFToIV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+}
+
+// Widening
+foreach mx = SchedMxListW in {
+  foreach sew = SchedSEWSet<mx, isF=0, isWidening=1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListW>.c;
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtIToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+  }
+}
+foreach mx = SchedMxListFW in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxListFW>.c;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+}
+foreach mx = SchedMxListFW in {
+  foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+    let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtFToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+// Narrowing
+foreach mx = SchedMxListW in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxListW>.c;
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVFNCvtFToIV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+}
+foreach mx = SchedMxListFW in {
+  foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtIToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtFToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// Worst case needs around 29/25/37 * LMUL cycles for f16/32/64.
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, 1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
+    defvar DivMicroOpLat =
+      !cond(!eq(sew, 16): 29, !eq(sew, 32): 25, /* SEW=64 */ true: 37);
+    defvar DivLatency = !mul(DivMicroOpLat, LMulLat);
+    let Latency = DivLatency, ReleaseAtCycles = [LMulLat, DivLatency] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFDivV",  [SiFiveP400VEXQ0, SiFiveP400VFloatDiv], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFDivF",  [SiFiveP400VEXQ0, SiFiveP400VFloatDiv], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSqrtV", [SiFiveP400VEXQ0, SiFiveP400VFloatDiv], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// 14. Vector Reduction Operations
+foreach mx = SchedMxList in {
+  foreach sew = SchedSEWSet<mx>.val in {
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+
+    // Simple reduction
+    defvar SimpleC = SimpleVIReduceCycles<mx>;
+    let Latency = SimpleC.latency, ReleaseAtCycles = [SimpleC.rthroughput] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVIRedV_From",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+
+    // Advanced reduction
+    defvar AdvancedC = AdvancedVIReduceCycles<sew, mx>;
+    let Latency = AdvancedC.latency, ReleaseAtCycles = [AdvancedC.rthroughput] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVIRedMinMaxV_From", [SiFiveP400VEXQ0],
+                                   mx, sew, IsWorstCase>;
+  }
+}
+
+foreach mx = SchedMxListWRed in {
+  foreach sew = SchedSEWSet<mx, 0, 1>.val in {
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListWRed>.c;
+    defvar SimpleC = SimpleVIReduceCycles<mx>;
+    let Latency = SimpleC.latency, ReleaseAtCycles = [SimpleC.rthroughput] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVIWRedV_From", [SiFiveP400VEXQ0],
+                                      mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, 1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
+
+    // Simple reduction.
+    defvar BaseC = VFReduceBaseCycles<sew>.val;
+    let Latency = !add(BaseC, !mul(6, !logtwo(LMulLat))), ReleaseAtCycles = [BaseC] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFRedV_From", [SiFiveP400VEXQ0],
+                                   mx, sew, IsWorstCase>;
+
+    // Advanced reduction.
+    defvar AdvancedC = AdvancedVFReduceCycles<sew, mx>;
+    let Latency = AdvancedC.latency, ReleaseAtCycles = [AdvancedC.rthroughput] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFRedMinMaxV_From",
+                                   [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+
+    defvar OrderedRedCycles = !mul(BaseC, LMulLat);
+    let Latency = OrderedRedCycles, ReleaseAtCycles = [OrderedRedCycles] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFRedOV_From", [SiFiveP400VEXQ0],
+                                    mx, sew, IsWorstCase>;
+  }
+}
+
+foreach mx = SchedMxListFWRed in {
+  foreach sew = SchedSEWSet<mx, 1, 1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListFWRed, 1>.c;
+    let Latency = !add(6, !mul(6, LMulLat)), ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedV_From",  [SiFiveP400VEXQ0],
+                                     mx, sew, IsWorstCase>;
+    }
+
+    defvar OrderedRedCycles = !mul(VFReduceBaseCycles<sew>.val, LMulLat);
+    let Latency = OrderedRedCycles, ReleaseAtCycles = [OrderedRedCycles] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedOV_From", [SiFiveP400VEXQ0],
+                                   mx, sew, IsWorstCase>;
+  }
+}
+
+// 15. Vector Mask Instructions
+foreach mx = SchedMxList in {
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [1] in {
+    defm "" : LMULWriteResMX<"WriteVMALUV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVMPopV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVMFFSV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVMSFSV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVIotaV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIdxV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// 16. Vector Permutation Instructions
+// Simple Slide
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSlideI",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVISlide1X", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFSlide1F", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+foreach mx = ["MF8", "MF4", "MF2", "M1"] in {
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [1] in {
+    defm "" : LMULWriteResMX<"WriteVSlideUpX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSlideDownX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// Complex Slide
+foreach mx = ["M2", "M4", "M8"] in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+
+  defvar UpLatAndCycles = !add(8, LMulLat);
+  let Latency = UpLatAndCycles, ReleaseAtCycles = [UpLatAndCycles] in {
+    defm "" : LMULWriteResMX<"WriteVSlideUpX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  defvar DownLatAndCycles = !add(8, !div(!mul(LMulLat, 3), 2));
+  let Latency = DownLatAndCycles, ReleaseAtCycles = [DownLatAndCycles] in {
+    defm "" : LMULWriteResMX<"WriteVSlideDownX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+let Latency = 2, ReleaseAtCycles = [2] in {
+  def : WriteRes<WriteVMovXS, [SiFiveP400VEXQ0]>;
+  def : WriteRes<WriteVMovSX, [SiFiveP400VEXQ0]>;
+}
+let Latency = 6, ReleaseAtCycles = [2] in {
+  def : WriteRes<WriteVMovFS, [SiFiveP400VEXQ0]>;
+  def : WriteRes<WriteVMovSF, [SiFiveP400VEXQ0]>;
+}
+
+// Simple Gather and Compress
+foreach mx = ["MF8", "MF4", "MF2", "M1"] in {
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 3, ReleaseAtCycles = [1] in {
+    defm "" : LMULWriteResMX<"WriteVRGatherVX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+foreach mx = ["MF8", "MF4", "MF2", "M1"] in {
+  foreach sew = SchedSEWSet<mx>.val in {
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+    let Latency = 3, ReleaseAtCycles = [1] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherVV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherEI16VV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVCompressV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// Complex Gather and Compress
+foreach mx = ["M2", "M4", "M8"] in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVRGatherVX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+foreach mx = ["M2", "M4", "M8"] in {
+  foreach sew = SchedSEWSet<mx>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+    let Latency = 6, ReleaseAtCycles = [!add(!mul(LMulLat, 2), 8)] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherVV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherEI16VV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVCompressV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// Simple Vrgather.vi
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVRGatherVI", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// Vector Crypto
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  // Zvbb
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVBREVV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVCLZV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVCPOPV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVCTZV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVWSLLV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVWSLLX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVWSLLI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvbc
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVCLMULV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVCLMULX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvkb
+  // VANDN uses WriteVIALU[V|X|I]
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVBREV8V",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVREV8V",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVRotV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVRotX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVRotI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvkg
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVGHSHV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVGMULV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // ZvknhaOrZvknhb
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSHA2CHV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSHA2CLV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSHA2MSV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvkned
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVAESMVV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAESKF1V", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAESKF2V", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAESZV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvksed
+  let Latency = 3, ReleaseAtCycles = [SiFiveP400VSM3CCycles<mx>.c] in
+  defm "" : LMULWriteResMX<"WriteVSM3CV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVSM3MEV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSM4KV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSM4RV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
 // Others
 def : WriteRes<WriteCSR, [SiFiveP400SYS]>;
 def : WriteRes<WriteNop, []>;
+def : WriteRes<WriteRdVLENB, [SiFiveP400SYS]>;
+
 
 // FIXME: This could be better modeled by looking at the regclasses of the operands.
 def : InstRW<[WriteIALU, ReadIALU], (instrs COPY)>;
@@ -356,6 +1012,217 @@ def : ReadAdvance<ReadSHXADD32, 0>;
 def : ReadAdvance<ReadSingleBit, 0>;
 def : ReadAdvance<ReadSingleBitImm, 0>;
 
+// 6. Configuration-Setting Instructions
+def : ReadAdvance<ReadVSETVLI, 0>;
+def : ReadAdvance<ReadVSETVL, 0>;
+
+// 7. Vector Loads and Stores
+def : ReadAdvance<ReadVLDX, 0>;
+def : ReadAdvance<ReadVSTX, 0>;
+defm "" : LMULReadAdvance<"ReadVSTEV", 0>;
+defm "" : LMULReadAdvance<"ReadVSTM", 0>;
+def : ReadAdvance<ReadVLDSX, 0>;
+def : ReadAdvance<ReadVSTSX, 0>;
+defm "" : LMULReadAdvance<"ReadVSTS8V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTS16V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTS32V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTS64V", 0>;
+defm "" : LMULReadAdvance<"ReadVLDUXV", 0>;
+defm "" : LMULReadAdvance<"ReadVLDOXV", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX8", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX16", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX32", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX64", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUXV", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX8V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX16V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX32V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX64V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX8", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX16", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX32", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX64", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOXV", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX8V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX16V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX32V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX64V", 0>;
+// LMUL Aware
+def : ReadAdvance<ReadVST1R, 0>;
+def : ReadAdvance<ReadVST2R, 0>;
+def : ReadAdvance<ReadVST4R, 0>;
+def : ReadAdvance<ReadVST8R, 0>;
+
+// 12. Vector Integer Arithmetic Instructions
+defm : LMULReadAdvance<"ReadVIALUV", 0>;
+defm : LMULReadAdvance<"ReadVIALUX", 0>;
+defm : LMULReadAdvanceW<"ReadVIWALUV", 0>;
+defm : LMULReadAdvanceW<"ReadVIWALUX", 0>;
+defm : LMULReadAdvance<"ReadVExtV", 0>;
+defm : LMULReadAdvance<"ReadVICALUV", 0>;
+defm : LMULReadAdvance<"ReadVICALUX", 0>;
+defm : LMULReadAdvance<"ReadVShiftV", 0>;
+defm : LMULReadAdvance<"ReadVShiftX", 0>;
+defm : LMULReadAdvanceW<"ReadVNShiftV", 0>;
+defm : LMULReadAdvanceW<"ReadVNShiftX", 0>;
+defm : LMULReadAdvance<"ReadVICmpV", 0>;
+defm : LMULReadAdvance<"ReadVICmpX", 0>;
+defm : LMULReadAdvance<"ReadVIMinMaxV", 0>;
+defm : LMULReadAdvance<"ReadVIMinMaxX", 0>;
+defm : LMULReadAdvance<"ReadVIMulV", 0>;
+defm : LMULReadAdvance<"ReadVIMulX", 0>;
+defm : LMULSEWReadAdvance<"ReadVIDivV", 0>;
+defm : LMULSEWReadAdvance<"ReadVIDivX", 0>;
+defm : LMULReadAdvanceW<"ReadVIWMulV", 0>;
+defm : LMULReadAdvanceW<"ReadVIWMulX", 0>;
+defm : LMULReadAdvance<"ReadVIMulAddV", 0>;
+defm : LMULReadAdvance<"ReadVIMulAddX", 0>;
+defm : LMULReadAdvanceW<"ReadVIWMulAddV", 0>;
+defm : LMULReadAdvanceW<"ReadVIWMulAddX", 0>;
+defm : LMULReadAdvance<"ReadVIMergeV", 0>;
+defm : LMULReadAdvance<"ReadVIMergeX", 0>;
+defm : LMULReadAdvance<"ReadVIMovV", 0>;
+defm : LMULReadAdvance<"ReadVIMovX", 0>;
+
+// 13. Vector Fixed-Point Arithmetic Instructions
+defm "" : LMULReadAdvance<"ReadVSALUV", 0>;
+defm "" : LMULReadAdvance<"ReadVSALUX", 0>;
+defm "" : LMULReadAdvance<"ReadVAALUV", 0>;
+defm "" : LMULReadAdvance<"ReadVAALUX", 0>;
+defm "" : LMULReadAdvance<"ReadVSMulV", 0>;
+defm "" : LMULReadAdvance<"ReadVSMulX", 0>;
+defm "" : LMULReadAdvance<"ReadVSShiftV", 0>;
+defm "" : LMULReadAdvance<"ReadVSShiftX", 0>;
+defm "" : LMULReadAdvanceW<"ReadVNClipV", 0>;
+defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
+
+// 14. Vector Floating-Point Instructions
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFRecpV", 0>;
+defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
+defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjF", 0>;
+defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
+defm "" : LMULReadAdvance<"ReadVFMergeV", 0>;
+defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
+defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
+defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
+defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
+defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
+defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
+
+// 15. Vector Reduction Operations
+def : ReadAdvance<ReadVIRedV, 0>;
+def : ReadAdvance<ReadVIRedV0, 0>;
+def : ReadAdvance<ReadVIWRedV, 0>;
+def : ReadAdvance<ReadVIWRedV0, 0>;
+def : ReadAdvance<ReadVFRedV, 0>;
+def : ReadAdvance<ReadVFRedV0, 0>;
+def : ReadAdvance<ReadVFRedOV, 0>;
+def : ReadAdvance<ReadVFRedOV0, 0>;
+def : ReadAdvance<ReadVFWRedV, 0>;
+def : ReadAdvance<ReadVFWRedV0, 0>;
+def : ReadAdvance<ReadVFWRedOV, 0>;
+def : ReadAdvance<ReadVFWRedOV0, 0>;
+
+// 16. Vector Mask Instructions
+defm "" : LMULReadAdvance<"ReadVMALUV", 0>;
+defm "" : LMULReadAdvance<"ReadVMPopV", 0>;
+defm "" : LMULReadAdvance<"ReadVMFFSV", 0>;
+defm "" : LMULReadAdvance<"ReadVMSFSV", 0>;
+defm "" : LMULReadAdvance<"ReadVIotaV", 0>;
+
+// 17. Vector Permutation Instructions
+def : ReadAdvance<ReadVMovXS, 0>;
+def : ReadAdvance<ReadVMovSX_V, 0>;
+def : ReadAdvance<ReadVMovSX_X, 0>;
+def : ReadAdvance<ReadVMovFS, 0>;
+def : ReadAdvance<ReadVMovSF_V, 0>;
+def : ReadAdvance<ReadVMovSF_F, 0>;
+defm "" : LMULReadAdvance<"ReadVISlideV", 0>;
+defm "" : LMULReadAdvance<"ReadVISlideX", 0>;
+defm "" : LMULReadAdvance<"ReadVFSlideV", 0>;
+defm "" : LMULReadAdvance<"ReadVFSlideF", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_data", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_index", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVRGatherEI16VV_data", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVRGatherEI16VV_index", 0>;
+defm "" : LMULReadAdvance<"ReadVRGatherVX_data", 0>;
+defm "" : LMULReadAdvance<"ReadVRGatherVX_index", 0>;
+defm "" : LMULReadAdvance<"ReadVRGatherVI_data", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVCompressV", 0>;
+// LMUL Aware
+def : ReadAdvance<ReadVMov1V, 0>;
+def : ReadAdvance<ReadVMov2V, 0>;
+def : ReadAdvance<ReadVMov4V, 0>;
+def : ReadAdvance<ReadVMov8V, 0>;
+
+// Others
+def : ReadAdvance<ReadVMask, 0>;
+def : ReadAdvance<ReadVPassthru_WorstCase, 0>;
+foreach mx = SchedMxList in {
+  def : ReadAdvance<!cast<SchedRead>("ReadVPassthru_" # mx), 0>;
+  foreach sew = SchedSEWSet<mx>.val in
+    def : ReadAdvance<!cast<SchedRead>("ReadVPassthru_" # mx  # "_E" # sew), 0>;
+}
+
+// Vector Crypto Extensions
+// Zvbb
+defm "" : LMULReadAdvance<"ReadVBREVV", 0>;
+defm "" : LMULReadAdvance<"ReadVCLZV", 0>;
+defm "" : LMULReadAdvance<"ReadVCPOPV", 0>;
+defm "" : LMULReadAdvance<"ReadVCTZV", 0>;
+defm "" : LMULReadAdvance<"ReadVWSLLV", 0>;
+defm "" : LMULReadAdvance<"ReadVWSLLX", 0>;
+// Zvbc
+defm "" : LMULReadAdvance<"ReadVCLMULV", 0>;
+defm "" : LMULReadAdvance<"ReadVCLMULX", 0>;
+// Zvkb
+// VANDN uses ReadVIALU[V|X|I]
+defm "" : LMULReadAdvance<"ReadVBREV8V", 0>;
+defm "" : LMULReadAdvance<"ReadVREV8V", 0>;
+defm "" : LMULReadAdvance<"ReadVRotV", 0>;
+defm "" : LMULReadAdvance<"ReadVRotX", 0>;
+// Zvkg
+defm "" : LMULReadAdvance<"ReadVGHSHV", 0>;
+defm "" : LMULReadAdvance<"ReadVGMULV", 0>;
+// Zvknha or Zvknhb
+defm "" : LMULReadAdvance<"ReadVSHA2CHV", 0>;
+defm "" : LMULReadAdvance<"ReadVSHA2CLV", 0>;
+defm "" : LMULReadAdvance<"ReadVSHA2MSV", 0>;
+// Zvkned
+defm "" : LMULReadAdvance<"ReadVAESMVV", 0>;
+defm "" : LMULReadAdvance<"ReadVAESKF1V", 0>;
+defm "" : LMULReadAdvance<"ReadVAESKF2V", 0>;
+defm "" : LMULReadAdvance<"ReadVAESZV", 0>;
+// Zvksed
+defm "" : LMULReadAdvance<"ReadVSM4KV", 0>;
+defm "" : LMULReadAdvance<"ReadVSM4RV", 0>;
+// Zbksh
+defm "" : LMULReadAdvance<"ReadVSM3CV", 0>;
+defm "" : LMULReadAdvance<"ReadVSM3MEV", 0>;
+
 //===----------------------------------------------------------------------===//
 // Unsupported extensions
 defm : UnsupportedSchedZabha;
@@ -364,7 +1231,5 @@ defm : UnsupportedSchedZbkb;
 defm : UnsupportedSchedZbkx;
 defm : UnsupportedSchedSFB;
 defm : UnsupportedSchedZfa;
-defm : UnsupportedSchedV;
 defm : UnsupportedSchedXsfvcp;
-defm : UnsupportedSchedZvk;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index b6884321f084112..794df2212dfa534 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -336,6 +336,7 @@ class RISCVPassConfig : public TargetPassConfig {
     if (TM.getOptLevel() != CodeGenOptLevel::None)
       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
     setEnableSinkAndFold(EnableSinkFold);
+    EnableLoopTermFold = true;
   }
 
   RISCVTargetMachine &getRISCVTargetMachine() const {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h b/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
index 05e61ac874abba2..ff7e3e4c752c330 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
@@ -22,7 +22,7 @@ class RISCVELFTargetObjectFile : public TargetLoweringObjectFileELF {
   MCSection *SmallROData16Section;
   MCSection *SmallROData32Section;
   MCSection *SmallBSSSection;
-  unsigned SSThreshold = 8;
+  unsigned SSThreshold = 0;
 
 public:
   unsigned getTextSectionAlignment() const override;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index f5eca2839acd058..cc69e1d118b5a18 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -394,9 +394,6 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
                      const TargetTransformInfo::LSRCost &C2);
 
-  bool shouldFoldTerminatingConditionAfterLSR() const {
-    return true;
-  }
   bool
   shouldConsiderAddressTypePromotion(const Instruction &I,
                                      bool &AllowPromotionWithoutCommonHeader);
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 979677ee92332dd..2abed1ac984e35b 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -65,6 +65,7 @@ class RISCVVectorPeephole : public MachineFunctionPass {
   bool convertToWholeRegister(MachineInstr &MI) const;
   bool convertToUnmasked(MachineInstr &MI) const;
   bool convertVMergeToVMv(MachineInstr &MI) const;
+  bool foldVMV_V_V(MachineInstr &MI);
 
   bool isAllOnesMask(const MachineInstr *MaskDef) const;
   std::optional<unsigned> getConstant(const MachineOperand &VL) const;
@@ -324,6 +325,143 @@ bool RISCVVectorPeephole::convertToUnmasked(MachineInstr &MI) const {
   return true;
 }
 
+/// Given two VL operands, returns the one known to be the smallest or nullptr
+/// if unknown.
+static const MachineOperand *getKnownMinVL(const MachineOperand *LHS,
+                                           const MachineOperand *RHS) {
+  if (LHS->isReg() && RHS->isReg() && LHS->getReg().isVirtual() &&
+      LHS->getReg() == RHS->getReg())
+    return LHS;
+  if (LHS->isImm() && LHS->getImm() == RISCV::VLMaxSentinel)
+    return RHS;
+  if (RHS->isImm() && RHS->getImm() == RISCV::VLMaxSentinel)
+    return LHS;
+  if (!LHS->isImm() || !RHS->isImm())
+    return nullptr;
+  return LHS->getImm() <= RHS->getImm() ? LHS : RHS;
+}
+
+/// Check if it's safe to move From down to To, checking that no physical
+/// registers are clobbered.
+static bool isSafeToMove(const MachineInstr &From, const MachineInstr &To) {
+  assert(From.getParent() == To.getParent() && !From.hasImplicitDef());
+  SmallVector<Register> PhysUses;
+  for (const MachineOperand &MO : From.all_uses())
+    if (MO.getReg().isPhysical())
+      PhysUses.push_back(MO.getReg());
+  bool SawStore = false;
+  for (auto II = From.getIterator(); II != To.getIterator(); II++) {
+    for (Register PhysReg : PhysUses)
+      if (II->definesRegister(PhysReg, nullptr))
+        return false;
+    if (II->mayStore()) {
+      SawStore = true;
+      break;
+    }
+  }
+  return From.isSafeToMove(SawStore);
+}
+
+static unsigned getSEWLMULRatio(const MachineInstr &MI) {
+  RISCVII::VLMUL LMUL = RISCVII::getLMul(MI.getDesc().TSFlags);
+  unsigned Log2SEW = MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
+  return RISCVVType::getSEWLMULRatio(1 << Log2SEW, LMUL);
+}
+
+/// If a PseudoVMV_V_V is the only user of its input, fold its passthru and VL
+/// into it.
+///
+/// %x = PseudoVADD_V_V_M1 %passthru, %a, %b, %vl1, sew, policy
+/// %y = PseudoVMV_V_V_M1 %passthru, %x, %vl2, sew, policy
+///
+/// ->
+///
+/// %y = PseudoVADD_V_V_M1 %passthru, %a, %b, min(vl1, vl2), sew, policy
+bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
+  if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMV_V_V)
+    return false;
+
+  MachineOperand &Passthru = MI.getOperand(1);
+
+  if (!MRI->hasOneUse(MI.getOperand(2).getReg()))
+    return false;
+
+  MachineInstr *Src = MRI->getVRegDef(MI.getOperand(2).getReg());
+  if (!Src || Src->hasUnmodeledSideEffects() ||
+      Src->getParent() != MI.getParent() || Src->getNumDefs() != 1 ||
+      !RISCVII::isFirstDefTiedToFirstUse(Src->getDesc()) ||
+      !RISCVII::hasVLOp(Src->getDesc().TSFlags) ||
+      !RISCVII::hasVecPolicyOp(Src->getDesc().TSFlags))
+    return false;
+
+  // Src needs to have the same VLMAX as MI
+  if (getSEWLMULRatio(MI) != getSEWLMULRatio(*Src))
+    return false;
+
+  // Src needs to have the same passthru as VMV_V_V
+  MachineOperand &SrcPassthru = Src->getOperand(1);
+  if (SrcPassthru.getReg() != RISCV::NoRegister &&
+      SrcPassthru.getReg() != Passthru.getReg())
+    return false;
+
+  // Because Src and MI have the same passthru, we can use either AVL as long as
+  // it's the smaller of the two.
+  //
+  // (src pt, ..., vl=5)       x x x x x|. . .
+  // (vmv.v.v pt, src, vl=3)   x x x|. . . . .
+  // ->
+  // (src pt, ..., vl=3)       x x x|. . . . .
+  //
+  // (src pt, ..., vl=3)       x x x|. . . . .
+  // (vmv.v.v pt, src, vl=6)   x x x . . .|. .
+  // ->
+  // (src pt, ..., vl=3)       x x x|. . . . .
+  MachineOperand &SrcVL = Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));
+  const MachineOperand *MinVL = getKnownMinVL(&MI.getOperand(3), &SrcVL);
+  if (!MinVL)
+    return false;
+
+  bool VLChanged = !MinVL->isIdenticalTo(SrcVL);
+  bool ActiveElementsAffectResult = RISCVII::activeElementsAffectResult(
+      TII->get(RISCV::getRVVMCOpcode(Src->getOpcode())).TSFlags);
+
+  if (VLChanged && (ActiveElementsAffectResult || Src->mayRaiseFPException()))
+    return false;
+
+  // If Src ends up using MI's passthru/VL, move it so it can access it.
+  // TODO: We don't need to do this if they already dominate Src.
+  if (!SrcVL.isIdenticalTo(*MinVL) || !SrcPassthru.isIdenticalTo(Passthru)) {
+    if (!isSafeToMove(*Src, MI))
+      return false;
+    Src->moveBefore(&MI);
+  }
+
+  if (SrcPassthru.getReg() != Passthru.getReg()) {
+    SrcPassthru.setReg(Passthru.getReg());
+    // If Src is masked then its passthru needs to be in VRNoV0.
+    if (Passthru.getReg() != RISCV::NoRegister)
+      MRI->constrainRegClass(Passthru.getReg(),
+                             TII->getRegClass(Src->getDesc(), 1, TRI,
+                                              *Src->getParent()->getParent()));
+  }
+
+  if (MinVL->isImm())
+    SrcVL.ChangeToImmediate(MinVL->getImm());
+  else if (MinVL->isReg())
+    SrcVL.ChangeToRegister(MinVL->getReg(), false);
+
+  // Use a conservative tu,mu policy, RISCVInsertVSETVLI will relax it if
+  // passthru is undef.
+  Src->getOperand(RISCVII::getVecPolicyOpNum(Src->getDesc()))
+      .setImm(RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED);
+
+  MRI->replaceRegWith(MI.getOperand(0).getReg(), Src->getOperand(0).getReg());
+  MI.eraseFromParent();
+  V0Defs.erase(&MI);
+
+  return true;
+}
+
 bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
@@ -358,11 +496,12 @@ bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
   }
 
   for (MachineBasicBlock &MBB : MF) {
-    for (MachineInstr &MI : MBB) {
+    for (MachineInstr &MI : make_early_inc_range(MBB)) {
       Changed |= convertToVLMAX(MI);
       Changed |= convertToUnmasked(MI);
       Changed |= convertToWholeRegister(MI);
       Changed |= convertVMergeToVMv(MI);
+      Changed |= foldVMV_V_V(MI);
     }
   }
 
diff --git a/llvm/lib/Target/Sparc/DelaySlotFiller.cpp b/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
index 7e129101fefc790..a74bfe490b90189 100644
--- a/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
+++ b/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
@@ -282,6 +282,20 @@ bool Filler::delayHasHazard(MachineBasicBlock::iterator candidate,
       Opcode >=  SP::FDIVD && Opcode <= SP::FSQRTD)
     return true;
 
+  if (Subtarget->fixTN0009() && candidate->mayStore())
+    return true;
+
+  if (Subtarget->fixTN0013()) {
+    switch (Opcode) {
+    case SP::FDIVS:
+    case SP::FDIVD:
+    case SP::FSQRTS:
+    case SP::FSQRTD:
+      return true;
+    default:
+      break;
+    }
+  }
 
   return false;
 }
diff --git a/llvm/lib/Target/Sparc/LeonFeatures.td b/llvm/lib/Target/Sparc/LeonFeatures.td
index 75273eff186867f..ac2ad5a577f4b3f 100644
--- a/llvm/lib/Target/Sparc/LeonFeatures.td
+++ b/llvm/lib/Target/Sparc/LeonFeatures.td
@@ -61,3 +61,38 @@ def FixAllFDIVSQRT : SubtargetFeature<
 def LeonCycleCounter
   : SubtargetFeature<"leoncyclecounter", "HasLeonCycleCounter", "true",
                      "Use the Leon cycle counter register">;
+
+def FixTN0009 : SubtargetFeature<
+  "fix-tn0009",
+  "FixTN0009",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0009"
+>;
+
+def FixTN0010 : SubtargetFeature<
+  "fix-tn0010",
+  "FixTN0010",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0010"
+>;
+
+def FixTN0011 : SubtargetFeature<
+  "fix-tn0011",
+  "FixTN0011",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0011"
+>;
+
+def FixTN0012 : SubtargetFeature<
+  "fix-tn0012",
+  "FixTN0012",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0012"
+>;
+
+def FixTN0013 : SubtargetFeature<
+  "fix-tn0013",
+  "FixTN0013",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0013"
+>;
diff --git a/llvm/lib/Target/Sparc/LeonPasses.cpp b/llvm/lib/Target/Sparc/LeonPasses.cpp
index 45a46c131d21eaf..018d8126f7a7e9a 100644
--- a/llvm/lib/Target/Sparc/LeonPasses.cpp
+++ b/llvm/lib/Target/Sparc/LeonPasses.cpp
@@ -19,6 +19,310 @@
 
 using namespace llvm;
 
+char ErrataWorkaround::ID = 0;
+
+ErrataWorkaround::ErrataWorkaround() : MachineFunctionPass(ID) {
+  initializeErrataWorkaroundPass(*PassRegistry::getPassRegistry());
+}
+
+INITIALIZE_PASS(ErrataWorkaround, "errata-workaround", "Errata workaround pass",
+                false, false)
+
+// Move iterator to the next instruction in the function, ignoring
+// meta instructions and inline assembly. Returns false when reaching
+// the end of the function.
+bool ErrataWorkaround::moveNext(MachineBasicBlock::iterator &I) {
+
+  MachineBasicBlock *MBB = I->getParent();
+
+  do {
+    I++;
+
+    while (I == MBB->end()) {
+      if (MBB->getFallThrough() == nullptr)
+        return false;
+      MBB = MBB->getFallThrough();
+      I = MBB->begin();
+    }
+  } while (I->isMetaInstruction() || I->isInlineAsm());
+
+  return true;
+}
+
+void ErrataWorkaround::insertNop(MachineBasicBlock::iterator I) {
+  BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(SP::NOP));
+}
+
+bool ErrataWorkaround::isFloat(MachineBasicBlock::iterator I) {
+  if (I->getNumOperands() == 0)
+    return false;
+
+  if (!I->getOperand(0).isReg())
+    return false;
+
+  unsigned reg = I->getOperand(0).getReg();
+
+  if (!SP::FPRegsRegClass.contains(reg) && !SP::DFPRegsRegClass.contains(reg))
+    return false;
+
+  return true;
+}
+
+bool ErrataWorkaround::isDivSqrt(MachineBasicBlock::iterator I) {
+  switch (I->getOpcode()) {
+  case SP::FDIVS:
+  case SP::FDIVD:
+  case SP::FSQRTS:
+  case SP::FSQRTD:
+    return true;
+  }
+  return false;
+}
+
+// Prevents the following code sequence from being generated:
+// (stb/sth/st/stf) -> (single non-store/load instruction) -> (any store)
+// If the sequence is detected a NOP instruction is inserted after
+// the first store instruction.
+bool ErrataWorkaround::checkSeqTN0009A(MachineBasicBlock::iterator I) {
+  switch (I->getOpcode()) {
+  case SP::STrr:
+  case SP::STri:
+  case SP::STBrr:
+  case SP::STBri:
+  case SP::STHrr:
+  case SP::STHri:
+  case SP::STFrr:
+  case SP::STFri:
+    break;
+  default:
+    return false;
+  }
+
+  MachineBasicBlock::iterator MI = I;
+  if (!moveNext(MI))
+    return false;
+
+  if (MI->mayStore() || MI->mayLoad())
+    return false;
+
+  MachineBasicBlock::iterator PatchHere = MI;
+
+  if (!moveNext(MI))
+    return false;
+
+  if (!MI->mayStore())
+    return false;
+
+  insertNop(PatchHere);
+  return true;
+}
+
+// Prevents the following code sequence from being generated:
+// (std/stdf) -> (any store)
+// If the sequence is detected a NOP instruction is inserted after
+// the first store instruction.
+bool ErrataWorkaround::checkSeqTN0009B(MachineBasicBlock::iterator I) {
+
+  switch (I->getOpcode()) {
+  case SP::STDrr:
+  case SP::STDri:
+  case SP::STDFrr:
+  case SP::STDFri:
+    break;
+  default:
+    return false;
+  }
+
+  MachineBasicBlock::iterator MI = I;
+
+  if (!moveNext(MI))
+    return false;
+
+  if (!MI->mayStore())
+    return false;
+
+  insertNop(MI);
+  return true;
+}
+
+// Insert a NOP at branch target if load in delay slot and atomic
+// instruction at branch target. Also insert a NOP between load
+// instruction and atomic instruction (swap or casa).
+bool ErrataWorkaround::checkSeqTN0010(MachineBasicBlock::iterator I) {
+
+  // Check for load instruction or branch bundled with load instruction
+  if (!I->mayLoad())
+    return false;
+
+  // Check for branch to atomic instruction with load in delay slot
+  if (I->isBranch()) {
+    MachineBasicBlock *TargetMBB = I->getOperand(0).getMBB();
+    MachineBasicBlock::iterator MI = TargetMBB->begin();
+
+    while (MI != TargetMBB->end() && MI->isMetaInstruction())
+      MI++;
+
+    if (MI == TargetMBB->end())
+      return false;
+
+    switch (MI->getOpcode()) {
+    case SP::SWAPrr:
+    case SP::SWAPri:
+    case SP::CASArr:
+      insertNop(MI);
+      break;
+    default:
+      break;
+    }
+  }
+
+  // Check for load followed by atomic instruction
+  MachineBasicBlock::iterator MI = I;
+  if (!moveNext(MI))
+    return false;
+
+  switch (MI->getOpcode()) {
+  case SP::SWAPrr:
+  case SP::SWAPri:
+  case SP::CASArr:
+    break;
+  default:
+    return false;
+  }
+  insertNop(MI);
+  return true;
+}
+
+// Do not allow functions to begin with an atomic instruction
+bool ErrataWorkaround::checkSeqTN0010First(MachineBasicBlock &MBB) {
+  MachineBasicBlock::iterator I = MBB.begin();
+  while (I != MBB.end() && I->isMetaInstruction())
+    I++;
+  switch (I->getOpcode()) {
+  case SP::SWAPrr:
+  case SP::SWAPri:
+  case SP::CASArr:
+    break;
+  default:
+    return false;
+  }
+  insertNop(I);
+  return true;
+}
+
+// Inserts a NOP instruction at the target of an integer branch if the
+// target is a floating-point instruction or floating-point branch.
+bool ErrataWorkaround::checkSeqTN0012(MachineBasicBlock::iterator I) {
+
+  if (I->getOpcode() != SP::BCOND && I->getOpcode() != SP::BCONDA)
+    return false;
+
+  MachineBasicBlock *TargetMBB = I->getOperand(0).getMBB();
+  MachineBasicBlock::iterator MI = TargetMBB->begin();
+
+  while (MI != TargetMBB->end() && MI->isMetaInstruction())
+    MI++;
+
+  if (MI == TargetMBB->end())
+    return false;
+
+  if (!isFloat(MI) && MI->getOpcode() != SP::FBCOND)
+    return false;
+
+  insertNop(MI);
+  return true;
+}
+
+// Prevents the following code sequence from being generated:
+// (div/sqrt) -> (2 to 3 floating-point operations or loads) -> (div/sqrt)
+// If the sequence is detected one or two NOP instruction are inserted after
+// the first div/sqrt instruction. No NOPs are inserted if one of the floating-
+// point instructions in the middle of the sequence is a (div/sqrt), or if
+// they have dependency on the destination register of the first (div/sqrt).
+//
+// The function also prevents the following code sequence from being generated,
+// (div/sqrt) -> (branch), by inserting a NOP instruction after the (div/sqrt).
+bool ErrataWorkaround::checkSeqTN0013(MachineBasicBlock::iterator I) {
+
+  if (!isDivSqrt(I))
+    return false;
+
+  unsigned dstReg = I->getOperand(0).getReg();
+
+  MachineBasicBlock::iterator MI = I;
+  if (!moveNext(MI))
+    return false;
+
+  if (MI->isBranch()) {
+    insertNop(MI);
+    return true;
+  }
+
+  MachineBasicBlock::iterator PatchHere = MI;
+
+  unsigned fpFound = 0;
+  for (unsigned i = 0; i < 4; i++) {
+
+    if (!isFloat(MI)) {
+      if (!moveNext(MI))
+        return false;
+      continue;
+    }
+
+    if (MI->readsRegister(dstReg, TRI))
+      return false;
+
+    if (isDivSqrt(MI)) {
+      if (i < 2)
+        return false;
+      if (fpFound < 2)
+        return false;
+
+      insertNop(PatchHere);
+      if (i == 2)
+        insertNop(PatchHere);
+      return true;
+    }
+
+    fpFound++;
+    if (!moveNext(MI))
+      return false;
+  }
+
+  return false;
+}
+
+bool ErrataWorkaround::runOnMachineFunction(MachineFunction &MF) {
+  bool Changed = false;
+  ST = &MF.getSubtarget<SparcSubtarget>();
+
+  if (!(ST->fixTN0009() || ST->fixTN0010() || ST->fixTN0012() ||
+        ST->fixTN0013()))
+    return false;
+
+  TII = ST->getInstrInfo();
+  TRI = ST->getRegisterInfo();
+
+  if (ST->fixTN0010())
+    Changed |= checkSeqTN0010First(MF.front());
+
+  for (auto &MBB : MF) {
+    for (auto &I : MBB) {
+      if (ST->fixTN0009()) {
+        Changed |= checkSeqTN0009A(I);
+        Changed |= checkSeqTN0009B(I);
+      }
+      if (ST->fixTN0010())
+        Changed |= checkSeqTN0010(I);
+      if (ST->fixTN0012())
+        Changed |= checkSeqTN0012(I);
+      if (ST->fixTN0013())
+        Changed |= checkSeqTN0013(I);
+    }
+  }
+  return Changed;
+}
+
 LEONMachineFunctionPass::LEONMachineFunctionPass(char &ID)
     : MachineFunctionPass(ID) {}
 
diff --git a/llvm/lib/Target/Sparc/LeonPasses.h b/llvm/lib/Target/Sparc/LeonPasses.h
index 9bc4569a12984ac..261b283fadaeb31 100644
--- a/llvm/lib/Target/Sparc/LeonPasses.h
+++ b/llvm/lib/Target/Sparc/LeonPasses.h
@@ -37,6 +37,33 @@ class LLVM_LIBRARY_VISIBILITY LEONMachineFunctionPass
   }
 };
 
+class LLVM_LIBRARY_VISIBILITY ErrataWorkaround : public MachineFunctionPass {
+protected:
+  const SparcSubtarget *ST;
+  const TargetInstrInfo *TII;
+  const TargetRegisterInfo *TRI;
+
+  bool checkSeqTN0009A(MachineBasicBlock::iterator I);
+  bool checkSeqTN0009B(MachineBasicBlock::iterator I);
+  bool checkSeqTN0010First(MachineBasicBlock &MBB);
+  bool checkSeqTN0010(MachineBasicBlock::iterator I);
+  bool checkSeqTN0012(MachineBasicBlock::iterator I);
+  bool checkSeqTN0013(MachineBasicBlock::iterator I);
+
+  bool moveNext(MachineBasicBlock::iterator &I);
+  bool isFloat(MachineBasicBlock::iterator I);
+  bool isDivSqrt(MachineBasicBlock::iterator I);
+  void insertNop(MachineBasicBlock::iterator I);
+
+public:
+  static char ID;
+
+  ErrataWorkaround();
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override { return "Errata workaround pass"; };
+};
+
 class LLVM_LIBRARY_VISIBILITY InsertNOPLoad : public LEONMachineFunctionPass {
 public:
   static char ID;
diff --git a/llvm/lib/Target/Sparc/Sparc.h b/llvm/lib/Target/Sparc/Sparc.h
index 33a803469603b3b..afcbc473295411e 100644
--- a/llvm/lib/Target/Sparc/Sparc.h
+++ b/llvm/lib/Target/Sparc/Sparc.h
@@ -32,6 +32,7 @@ FunctionPass *createSparcDelaySlotFillerPass();
 void LowerSparcMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
                                     AsmPrinter &AP);
 void initializeSparcDAGToDAGISelLegacyPass(PassRegistry &);
+void initializeErrataWorkaroundPass(PassRegistry &);
 } // namespace llvm
 
 namespace llvm {
diff --git a/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp b/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
index 6855471840e9dbf..87d76ebf268c41f 100644
--- a/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
+++ b/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
@@ -50,8 +50,7 @@ namespace {
     StringRef getPassName() const override { return "Sparc Assembly Printer"; }
 
     void printOperand(const MachineInstr *MI, int opNum, raw_ostream &OS);
-    void printMemOperand(const MachineInstr *MI, int opNum, raw_ostream &OS,
-                         const char *Modifier = nullptr);
+    void printMemOperand(const MachineInstr *MI, int opNum, raw_ostream &OS);
 
     void emitFunctionBodyStart() override;
     void emitInstruction(const MachineInstr *MI) override;
@@ -277,6 +276,12 @@ void SparcAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case TargetOpcode::DBG_VALUE:
     // FIXME: Debug Value.
     return;
+  case SP::CASArr:
+  case SP::SWAPrr:
+  case SP::SWAPri:
+    if (MF->getSubtarget<SparcSubtarget>().fixTN0011())
+      OutStreamer->emitCodeAlignment(Align(16), &getSubtargetInfo());
+    break;
   case SP::GETPCX:
     LowerGETPCXAndEmitMCInsts(MI, getSubtargetInfo());
     return;
@@ -401,16 +406,9 @@ void SparcAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
 }
 
 void SparcAsmPrinter::printMemOperand(const MachineInstr *MI, int opNum,
-                                      raw_ostream &O, const char *Modifier) {
+                                      raw_ostream &O) {
   printOperand(MI, opNum, O);
 
-  // If this is an ADD operand, emit it like normal operands.
-  if (Modifier && !strcmp(Modifier, "arith")) {
-    O << ", ";
-    printOperand(MI, opNum+1, O);
-    return;
-  }
-
   if (MI->getOperand(opNum+1).isReg() &&
       MI->getOperand(opNum+1).getReg() == SP::G0)
     return;   // don't print "+%g0"
diff --git a/llvm/lib/Target/Sparc/SparcTargetMachine.cpp b/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
index ea403230bea8938..fec2d3a35ae6d29 100644
--- a/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
+++ b/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
@@ -29,6 +29,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSparcTarget() {
 
   PassRegistry &PR = *PassRegistry::getPassRegistry();
   initializeSparcDAGToDAGISelLegacyPass(PR);
+  initializeErrataWorkaroundPass(PR);
 }
 
 static cl::opt<bool>
@@ -193,6 +194,7 @@ void SparcPassConfig::addPreEmitPass(){
   addPass(new InsertNOPLoad());
   addPass(new DetectRoundChange());
   addPass(new FixAllFDIVSQRT());
+  addPass(new ErrataWorkaround());
 }
 
 void SparcV8TargetMachine::anchor() { }
diff --git a/llvm/lib/Target/VE/VECustomDAG.cpp b/llvm/lib/Target/VE/VECustomDAG.cpp
index 873ffea498de839..2855a65f654c96e 100644
--- a/llvm/lib/Target/VE/VECustomDAG.cpp
+++ b/llvm/lib/Target/VE/VECustomDAG.cpp
@@ -410,7 +410,7 @@ SDValue VECustomDAG::getConstantMask(Packing Packing, bool AllTrue) const {
   auto MaskVT = getLegalVectorType(Packing, MVT::i1);
 
   // VEISelDAGtoDAG will replace this pattern with the constant-true VM.
-  auto TrueVal = DAG.getConstant(-1, DL, MVT::i32);
+  auto TrueVal = DAG.getAllOnesConstant(DL, MVT::i32);
   auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);
   auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});
   if (AllTrue)
diff --git a/llvm/lib/Target/VE/VEInstrInfo.td b/llvm/lib/Target/VE/VEInstrInfo.td
index 75ef3b7336dbd5d..eb6a85298007935 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.td
+++ b/llvm/lib/Target/VE/VEInstrInfo.td
@@ -44,8 +44,8 @@ def ULO7 : SDNodeXForm<imm, [{
                                    SDLoc(N), MVT::i32);
 }]>;
 def LO7 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(SignExtend32(N->getSExtValue(), 7),
-                                   SDLoc(N), MVT::i32);
+  return CurDAG->getSignedConstant(SignExtend64(N->getSExtValue(), 7),
+                                   SDLoc(N), MVT::i32, /*isTarget=*/true);
 }]>;
 def MIMM : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(val2MImm(getImmVal(N)),
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 8404f2231680d61..d850f4fd7683118 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/WinEHFuncInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/EHPersonalities.h"
@@ -2616,7 +2617,7 @@ StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
   // object.
   // We need to factor in additional offsets applied during the prologue to the
   // frame, base, and stack pointer depending on which is used.
-  int Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea();
+  int64_t Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea();
   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   unsigned CSSize = X86FI->getCalleeSavedFrameSize();
   uint64_t StackSize = MFI.getStackSize();
@@ -4140,6 +4141,14 @@ void X86FrameLowering::processFunctionBeforeFrameFinalized(
   // emitPrologue if it gets called and emits CFI.
   MF.setHasWinCFI(false);
 
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  // If the frame is big enough that we might need to scavenge a register to
+  // handle huge offsets, reserve a stack slot for that now.
+  if (!isInt<32>(MFI.estimateStackSize(MF))) {
+    int FI = MFI.CreateStackObject(SlotSize, Align(SlotSize), false);
+    RS->addScavengingFrameIndex(FI);
+  }
+
   // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
   // aligned. The format doesn't support misaligned stack adjustments.
   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 74804e5c9783dec..d0a54ab8993c267 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -314,7 +314,8 @@ namespace {
         Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
                                              AM.SymbolFlags);
       else
-        Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32);
+        Disp =
+            CurDAG->getSignedConstant(AM.Disp, DL, MVT::i32, /*isTarget=*/true);
 
       if (AM.Segment.getNode())
         Segment = AM.Segment;
@@ -2130,7 +2131,7 @@ static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N,
     X = NewX;
   }
 
-  SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT);
+  SDValue NewMask = DAG.getSignedConstant(Mask >> ShiftAmt, DL, VT);
   SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask);
   SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1));
 
@@ -3733,7 +3734,8 @@ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
       }
 
       if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
-        Operand = CurDAG->getTargetConstant(OperandV, SDLoc(Node), MemVT);
+        Operand = CurDAG->getSignedConstant(OperandV, SDLoc(Node), MemVT,
+                                            /*isTarget=*/true);
         NewOpc = SelectImmOpcode(Opc);
       }
     }
@@ -4507,7 +4509,7 @@ bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) {
     X = NewX;
   }
 
-  SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
+  SDValue NewCst = CurDAG->getSignedConstant(ShiftedVal, dl, NVT);
   insertDAGNode(*CurDAG, SDValue(N, 0), NewCst);
   SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT, X, NewCst);
   insertDAGNode(*CurDAG, SDValue(N, 0), NewBinOp);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 11c9a992cbdee92..169c955f0ba89f0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -10005,8 +10005,8 @@ static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
 
 // X86 has dedicated unpack instructions that can handle specific blend
 // operations: UNPCKH and UNPCKL.
-static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
-                                     ArrayRef<int> Mask, SDValue V1, SDValue V2,
+static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, SDValue V1,
+                                     SDValue V2, ArrayRef<int> Mask,
                                      SelectionDAG &DAG) {
   SmallVector<int, 8> Unpckl;
   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);
@@ -10032,9 +10032,9 @@ static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
 
 /// Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit)
 /// followed by unpack 256-bit.
-static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,
-                                        ArrayRef<int> Mask, SDValue V1,
-                                        SDValue V2, SelectionDAG &DAG) {
+static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, SDValue V1,
+                                        SDValue V2, ArrayRef<int> Mask,
+                                        SelectionDAG &DAG) {
   SmallVector<int, 32> Unpckl, Unpckh;
   createSplat2ShuffleMask(VT, Unpckl, /* Lo */ true);
   createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false);
@@ -10432,9 +10432,10 @@ static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
   return false;
 }
 
-static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
-                                    SDValue V1, SDValue V2, SelectionDAG &DAG,
-                                    const X86Subtarget &Subtarget) {
+static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, SDValue V1,
+                                    SDValue V2, ArrayRef<int> Mask,
+                                    const X86Subtarget &Subtarget,
+                                    SelectionDAG &DAG) {
   MVT PackVT;
   unsigned PackOpcode;
   unsigned SizeBits = VT.getSizeInBits();
@@ -10763,9 +10764,9 @@ static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
     for (int i = 0, Size = Mask.size(); i < Size; ++i)
       for (int j = 0; j < Scale; ++j)
         VSELECTMask.push_back(
-            Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)
-                        : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL,
-                                          MVT::i8));
+            Mask[i] < 0
+                ? DAG.getUNDEF(MVT::i8)
+                : DAG.getSignedConstant(Mask[i] < Size ? -1 : 0, DL, MVT::i8));
 
     V1 = DAG.getBitcast(BlendVT, V1);
     V2 = DAG.getBitcast(BlendVT, V2);
@@ -11156,7 +11157,7 @@ static bool isSingleElementRepeatedMask(ArrayRef<int> Mask) {
 /// blends. For vXi8/vXi16 shuffles we may use unpack instead of blend.
 static SDValue lowerShuffleAsDecomposedShuffleMerge(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
-    const X86Subtarget &Subtarget, SelectionDAG &DAG) {
+    const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
   int NumElts = Mask.size();
   int NumLanes = VT.getSizeInBits() / 128;
   int NumEltsPerLane = NumElts / NumLanes;
@@ -11164,6 +11165,7 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
   // Shuffle the input elements into the desired positions in V1 and V2 and
   // unpack/blend them together.
   bool IsAlternating = true;
+  bool V1Zero = true, V2Zero = true;
   SmallVector<int, 32> V1Mask(NumElts, -1);
   SmallVector<int, 32> V2Mask(NumElts, -1);
   SmallVector<int, 32> FinalMask(NumElts, -1);
@@ -11172,10 +11174,12 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
     if (M >= 0 && M < NumElts) {
       V1Mask[i] = M;
       FinalMask[i] = i;
+      V1Zero &= Zeroable[i];
       IsAlternating &= (i & 1) == 0;
     } else if (M >= NumElts) {
       V2Mask[i] = M - NumElts;
       FinalMask[i] = i + NumElts;
+      V2Zero &= Zeroable[i];
       IsAlternating &= (i & 1) == 1;
     }
   }
@@ -11228,7 +11232,7 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
     // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2, t4
     // it is better to process t4 first to create a vector of t4[0], then unpack
     // that vector with t2.
-    if (!isSingleElementRepeatedMask(V1Mask) &&
+    if (!V1Zero && !V2Zero && !isSingleElementRepeatedMask(V1Mask) &&
         !isSingleElementRepeatedMask(V2Mask))
       if (SDValue UnpackPerm =
               lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG))
@@ -12858,7 +12862,7 @@ static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, V1, V2, Mask, DAG))
     return V;
 
   unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
@@ -12935,7 +12939,7 @@ static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, V1, V2, Mask, DAG))
     return V;
 
   // Try to use byte rotation instructions.
@@ -12955,7 +12959,7 @@ static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // a permute. That will be faster than the domain cross.
   if (IsBlendSupported)
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v2i64, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // We implement this with SHUFPD which is pretty lame because it will likely
   // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
@@ -13157,7 +13161,7 @@ static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   }
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, V1, V2, Mask, DAG))
     return V;
 
   // Otherwise fall back to a SHUFPS lowering strategy.
@@ -13250,7 +13254,7 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, V1, V2, Mask, DAG))
     return V;
 
   // Try to use byte rotation instructions.
@@ -13274,7 +13278,7 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     // a permute. That will be faster than the domain cross.
     if (IsBlendSupported)
       return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i32, V1, V2, Mask,
-                                                  Subtarget, DAG);
+                                                  Zeroable, Subtarget, DAG);
 
     // Try to lower by permuting the inputs into an unpack instruction.
     if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
@@ -13905,12 +13909,12 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return Rotate;
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
       return V;
 
     // Use dedicated pack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
-                                         Subtarget))
+    if (SDValue V =
+            lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
       return V;
 
     // Try to use byte rotation instructions.
@@ -13959,12 +13963,12 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use lower using a truncation.
@@ -14065,8 +14069,8 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   // We can always bit-blend if we have to so the fallback strategy is to
   // decompose into single-input permutes and blends/unpacks.
-  return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2,
-                                              Mask, Subtarget, DAG);
+  return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2, Mask,
+                                              Zeroable, Subtarget, DAG);
 }
 
 /// Lower 8-lane 16-bit floating point shuffles.
@@ -14165,8 +14169,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Rotate;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use a zext lowering.
@@ -14203,7 +14207,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                                  Subtarget, DAG))
       return Rotate;
 
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
       return V;
 
     // Check whether we can widen this to an i16 shuffle by duplicating bytes.
@@ -14307,7 +14311,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
     return V;
 
   // Try to use byte shift instructions to mask.
@@ -14444,7 +14448,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Handle multi-input cases by blending/unpacking single-input shuffles.
   if (NumV2Elements > 0)
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v16i8, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // The fallback path for single-input shuffles widens this into two v8i16
   // vectors with unpacks, shuffles those, and then pulls them back together
@@ -14668,6 +14672,7 @@ static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
 /// results.
 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
                                           SDValue V2, ArrayRef<int> Mask,
+                                          const APInt &Zeroable,
                                           const X86Subtarget &Subtarget,
                                           SelectionDAG &DAG) {
   assert(!V2.isUndef() && "This routine must not be used to lower single-input "
@@ -14694,8 +14699,8 @@ static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
     return true;
   };
   if (DoBothBroadcast())
-    return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget,
-                                                DAG);
+    return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Zeroable,
+                                                Subtarget, DAG);
 
   // If the inputs all stem from a single 128-bit lane of each input, then we
   // split them rather than blending because the split will decompose to
@@ -14714,8 +14719,8 @@ static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
 
   // Otherwise, just fall back to decomposed shuffles and a blend/unpack. This
   // requires that the decomposed single-input shuffles don't end up here.
-  return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget,
-                                              DAG);
+  return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Zeroable,
+                                              Subtarget, DAG);
 }
 
 // Lower as SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
@@ -15878,7 +15883,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   }
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, V1, V2, Mask, DAG))
     return V;
 
   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
@@ -15907,7 +15912,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // blend the result.
   if (V1IsInPlace || V2IsInPlace)
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
   // results into the target lanes.
@@ -15934,10 +15939,10 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // can fully permute the elements.
   if (Subtarget.hasAVX2())
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // Otherwise fall back on generic lowering.
-  return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
                                     Subtarget, DAG);
 }
 
@@ -16017,7 +16022,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Rotate;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, V1, V2, Mask, DAG))
     return V;
 
   bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
@@ -16027,7 +16032,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // blend the result.
   if (V1IsInPlace || V2IsInPlace)
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
   // results into the target lanes.
@@ -16051,7 +16056,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   // Otherwise fall back on generic blend lowering.
   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
-                                              Subtarget, DAG);
+                                              Zeroable, Subtarget, DAG);
 }
 
 /// Handle lowering of 8-lane 32-bit floating point shuffles.
@@ -16106,7 +16111,7 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, V1, V2, Mask, DAG))
       return V;
 
     // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
@@ -16162,17 +16167,17 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // since after split we get a more efficient code using vpunpcklwd and
   // vpunpckhwd instrs than vblend.
   if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG))
-    return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Subtarget,
-                                      DAG);
+    return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Zeroable,
+                                      Subtarget, DAG);
 
   // If we have AVX2 then we always want to lower with a blend because at v8 we
   // can fully permute the elements.
   if (Subtarget.hasAVX2())
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8f32, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // Otherwise fall back on generic lowering.
-  return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Zeroable,
                                     Subtarget, DAG);
 }
 
@@ -16210,8 +16215,8 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // vpunpcklwd and vpunpckhwd instrs.
   if (isUnpackWdShuffleMask(Mask, MVT::v8i32, DAG) && !V2.isUndef() &&
       !Subtarget.hasAVX512())
-    return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Subtarget,
-                                      DAG);
+    return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Zeroable,
+                                      Subtarget, DAG);
 
   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
                                           Zeroable, Subtarget, DAG))
@@ -16247,7 +16252,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, V1, V2, Mask, DAG))
       return V;
   }
 
@@ -16287,7 +16292,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (V2.isUndef()) {
     // Try to produce a fixed cross-128-bit lane permute followed by unpack
     // because that should be faster than the variable permute alternatives.
-    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, V1, V2, Mask, DAG))
       return V;
 
     // If the shuffle patterns aren't repeated but it's a single input, directly
@@ -16315,7 +16320,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   // Otherwise fall back on generic blend lowering.
   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i32, V1, V2, Mask,
-                                              Subtarget, DAG);
+                                              Zeroable, Subtarget, DAG);
 }
 
 /// Handle lowering of 16-lane 16-bit integer shuffles.
@@ -16348,12 +16353,12 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use lower using a truncation.
@@ -16386,7 +16391,7 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
     // Try to produce a fixed cross-128-bit lane permute followed by unpack
     // because that should be faster than the variable permute alternatives.
-    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
       return V;
 
     // There are no generalized cross-lane shuffle operations available on i16
@@ -16437,7 +16442,7 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return V;
 
   // Otherwise fall back on generic lowering.
-  return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
                                     Subtarget, DAG);
 }
 
@@ -16471,12 +16476,12 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use lower using a truncation.
@@ -16512,7 +16517,7 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {
     // Try to produce a fixed cross-128-bit lane permute followed by unpack
     // because that should be faster than the variable permute alternatives.
-    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, V1, V2, Mask, DAG))
       return V;
 
     if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
@@ -16558,7 +16563,7 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return V;
 
   // Otherwise fall back on generic lowering.
-  return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
                                     Subtarget, DAG);
 }
 
@@ -16779,7 +16784,7 @@ static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                            V2, Subtarget, DAG))
     return Shuf128;
 
-  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
+  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, V1, V2, Mask, DAG))
     return Unpck;
 
   // Check if the blend happens to exactly fit that of SHUFPD.
@@ -16824,7 +16829,7 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, V1, V2, Mask, DAG))
       return V;
 
     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
@@ -16923,7 +16928,7 @@ static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                                   Subtarget, DAG))
       return Rotate;
 
-  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
+  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, V1, V2, Mask, DAG))
     return Unpck;
 
   // If we have AVX512F support, we can use VEXPAND.
@@ -16981,7 +16986,7 @@ static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, V1, V2, Mask, DAG))
       return V;
   }
 
@@ -17053,12 +17058,12 @@ static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return ZExt;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
   if (SDValue V =
-          lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget))
+          lowerShuffleWithPACK(DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use shift instructions.
@@ -17117,12 +17122,12 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return ZExt;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use shift instructions.
@@ -18650,7 +18655,7 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
   // addition for it.
   if (Offset != 0)
     Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result,
-                         DAG.getConstant(Offset, dl, PtrVT));
+                         DAG.getSignedConstant(Offset, dl, PtrVT));
 
   return Result;
 }
@@ -23595,7 +23600,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
 
       if (!FlipSigns && !Invert && ISD::isBuildVectorAllOnes(Op1.getNode())) {
         Op0 = DAG.getBitcast(MVT::v4i32, Op0);
-        Op1 = DAG.getConstant(-1, dl, MVT::v4i32);
+        Op1 = DAG.getAllOnesConstant(dl, MVT::v4i32);
 
         SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
         static const int MaskHi[] = { 1, 1, 3, 3 };
@@ -24387,7 +24392,7 @@ static SDValue LowerSIGN_EXTEND_Mask(SDValue Op, const SDLoc &dl,
       (Subtarget.hasBWI() && WideEltVT.getSizeInBits() <= 16)) {
     V = DAG.getNode(Op.getOpcode(), dl, WideVT, In);
   } else {
-    SDValue NegOne = DAG.getConstant(-1, dl, WideVT);
+    SDValue NegOne = DAG.getAllOnesConstant(dl, WideVT);
     SDValue Zero = DAG.getConstant(0, dl, WideVT);
     V = DAG.getSelect(dl, WideVT, In, NegOne, Zero);
   }
@@ -24829,14 +24834,14 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 
       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                         Overflow);
+                         Overflow, Op->getFlags());
     }
 
     if (LHS.getSimpleValueType().isInteger()) {
       SDValue CCVal;
       SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                         EFLAGS);
+                         EFLAGS, Op->getFlags());
     }
 
     if (CC == ISD::SETOEQ) {
@@ -24862,10 +24867,10 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
               DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
           SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
           Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest,
-                              CCVal, Cmp);
+                              CCVal, Cmp, Op->getFlags());
           CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
           return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                             Cmp);
+                             Cmp, Op->getFlags());
         }
       }
     } else if (CC == ISD::SETUNE) {
@@ -24874,18 +24879,18 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
       // separate test.
       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
       SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
-      Chain =
-          DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp);
+      Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
+                          Cmp, Op->getFlags());
       CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                         Cmp);
+                         Cmp, Op->getFlags());
     } else {
       X86::CondCode X86Cond =
           TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG);
       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                         Cmp);
+                         Cmp, Op->getFlags());
     }
   }
 
@@ -24896,7 +24901,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 
     SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
     return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                       Overflow);
+                       Overflow, Op->getFlags());
   }
 
   // Look past the truncate if the high bits are known zero.
@@ -24915,8 +24920,8 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 
   SDValue CCVal;
   SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
-  return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                     EFLAGS);
+  return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, EFLAGS,
+                     Op->getFlags());
 }
 
 // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
@@ -24971,9 +24976,9 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
       Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
     }
     if (Alignment && *Alignment > StackAlign)
-      Result =
-          DAG.getNode(ISD::AND, dl, VT, Result,
-                      DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
+      Result = DAG.getNode(
+          ISD::AND, dl, VT, Result,
+          DAG.getSignedConstant(~(Alignment->value() - 1ULL), dl, VT));
     Chain = DAG.getCopyToReg(Chain, dl, SPReg, Result); // Output chain
   } else if (SplitStack) {
     MachineRegisterInfo &MRI = MF.getRegInfo();
@@ -25005,8 +25010,9 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
     Chain = SP.getValue(1);
 
     if (Alignment) {
-      SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                       DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
+      SP = DAG.getNode(
+          ISD::AND, dl, VT, SP.getValue(0),
+          DAG.getSignedConstant(~(Alignment->value() - 1ULL), dl, VT));
       Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
     }
 
@@ -26090,7 +26096,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                           Op.getOperand(3));
       } else {
         SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
-                                    DAG.getConstant(-1, dl, MVT::i8));
+                                    DAG.getAllOnesConstant(dl, MVT::i8));
         Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2),
                           Op.getOperand(3), GenCF.getValue(1));
       }
@@ -28437,13 +28443,27 @@ static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
   bool IsSigned = Op.getOpcode() == ISD::ABDS;
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
-  // TODO: Move to TargetLowering expandABD() once we have ABD promotion.
   if (VT.isScalarInteger()) {
+    // abds(lhs, rhs) -> select(slt(lhs,rhs),sub(rhs,lhs),sub(lhs,rhs))
+    // abdu(lhs, rhs) -> select(ult(lhs,rhs),sub(rhs,lhs),sub(lhs,rhs))
+    if (Subtarget.canUseCMOV() && VT.bitsGE(MVT::i32)) {
+      SDVTList VTs = DAG.getVTList(VT, MVT::i32);
+      X86::CondCode CC = IsSigned ? X86::COND_L : X86::COND_B;
+      SDValue LHS = DAG.getFreeze(Op.getOperand(0));
+      SDValue RHS = DAG.getFreeze(Op.getOperand(1));
+      SDValue Diff0 = DAG.getNode(X86ISD::SUB, dl, VTs, LHS, RHS);
+      SDValue Diff1 = DAG.getNode(X86ISD::SUB, dl, VTs, RHS, LHS);
+      return DAG.getNode(X86ISD::CMOV, dl, VT, Diff1, Diff0,
+                         DAG.getTargetConstant(CC, dl, MVT::i8),
+                         Diff1.getValue(1));
+    }
+
+    // TODO: Move to TargetLowering expandABD() once we have ABD promotion.
+    // abds(lhs, rhs) -> trunc(abs(sub(sext(lhs), sext(rhs))))
+    // abdu(lhs, rhs) -> trunc(abs(sub(zext(lhs), zext(rhs))))
     unsigned WideBits = std::max<unsigned>(2 * VT.getScalarSizeInBits(), 32u);
     MVT WideVT = MVT::getIntegerVT(WideBits);
     if (TLI.isTypeLegal(WideVT)) {
-      // abds(lhs, rhs) -> trunc(abs(sub(sext(lhs), sext(rhs))))
-      // abdu(lhs, rhs) -> trunc(abs(sub(zext(lhs), zext(rhs))))
       unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
       SDValue LHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(0));
       SDValue RHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(1));
@@ -29373,7 +29393,7 @@ static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG,
 
         // Create the mask using vXi16 shifts. For shift-rights we need to move
         // the upper byte down before splatting the vXi8 mask.
-        SDValue BitMask = DAG.getConstant(-1, dl, ExtVT);
+        SDValue BitMask = DAG.getAllOnesConstant(dl, ExtVT);
         BitMask = getTargetVShiftNode(LogicalX86Op, dl, ExtVT, BitMask,
                                       BaseShAmt, BaseShAmtIdx, Subtarget, DAG);
         if (Opcode != ISD::SHL)
@@ -36203,6 +36223,7 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
       MIB.add(MO);
   }
   MIB.setMemRefs(MMOs);
+  MIB.setMIFlag(MachineInstr::FrameDestroy);
 
   // Reload IP
   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), Tmp);
@@ -36228,6 +36249,7 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
                                  // the last instruction of the expansion.
   }
   MIB.setMemRefs(MMOs);
+  MIB.setMIFlag(MachineInstr::FrameDestroy);
 
   // Jump
   BuildMI(*thisMBB, MI, MIMD, TII->get(IJmpOpc)).addReg(Tmp);
@@ -42520,6 +42542,26 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       SDValue Insert =
           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
       return TLO.CombineTo(Op, Insert);
+    }
+      // Conversions.
+      // TODO: Add more CVT opcodes when we have test coverage.
+    case X86ISD::CVTTP2SI:
+    case X86ISD::CVTTP2UI:
+    case X86ISD::CVTPH2PS: {
+      SDLoc DL(Op);
+      unsigned Scale = SizeInBits / ExtSizeInBits;
+      SDValue SrcOp = Op.getOperand(0);
+      MVT SrcVT = SrcOp.getSimpleValueType();
+      unsigned SrcExtSize =
+          std::max<unsigned>(SrcVT.getSizeInBits() / Scale, 128);
+      MVT ExtVT = MVT::getVectorVT(VT.getSimpleVT().getScalarType(),
+                                   ExtSizeInBits / VT.getScalarSizeInBits());
+      SDValue ExtOp = TLO.DAG.getNode(
+          Opc, DL, ExtVT, extractSubVector(SrcOp, 0, TLO.DAG, DL, SrcExtSize));
+      SDValue UndefVec = TLO.DAG.getUNDEF(VT);
+      SDValue Insert =
+          insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
+      return TLO.CombineTo(Op, Insert);
     }
       // Zero upper elements.
     case X86ISD::VZEXT_MOVL:
@@ -50555,7 +50597,7 @@ static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
     // X - SETAE --> adc X, -1
     return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
                        DAG.getVTList(VT, MVT::i32), X,
-                       DAG.getConstant(-1, DL, VT), EFLAGS);
+                       DAG.getAllOnesConstant(DL, VT), EFLAGS);
   }
 
   if (CC == X86::COND_BE) {
@@ -50576,7 +50618,7 @@ static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
       SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
       return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
                          DAG.getVTList(VT, MVT::i32), X,
-                         DAG.getConstant(-1, DL, VT), NewEFLAGS);
+                         DAG.getAllOnesConstant(DL, VT), NewEFLAGS);
     }
   }
 
@@ -50635,7 +50677,7 @@ static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
   // X + (Z != 0) --> add X, (zext(setne Z, 0)) --> sbb X, -1, (cmp Z, 1)
   if (CC == X86::COND_NE)
     return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
-                       DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1));
+                       DAG.getAllOnesConstant(DL, VT), Cmp1.getValue(1));
 
   // X - (Z == 0) --> sub X, (zext(sete  Z, 0)) --> sbb X, 0, (cmp Z, 1)
   // X + (Z == 0) --> add X, (zext(sete  Z, 0)) --> adc X, 0, (cmp Z, 1)
@@ -50919,8 +50961,9 @@ static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(),
                                                *DAG.getContext(), ResultType);
-  SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp,
-                              DAG.getConstant(-1, DL, ShiftOpTy), ISD::SETGT);
+  SDValue Cond =
+      DAG.getSetCC(DL, SetCCResultType, ShiftOp,
+                   DAG.getAllOnesConstant(DL, ShiftOpTy), ISD::SETGT);
   if (SetCCResultType != ResultType)
     Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond);
   return Cond;
@@ -57179,11 +57222,13 @@ static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
       }
     }
     // v4i32 CVTPS2DQ(v4f32) / CVTPS2UDQ(v4f32).
-    if ((InOpcode == ISD::FP_TO_SINT ||
-         (InOpcode == ISD::FP_TO_UINT && Subtarget.hasVLX())) &&
-        VT == MVT::v4i32) {
+    // v4f32 CVTDQ2PS(v4i32) / CVTUDQ2PS(v4i32).
+    if ((InOpcode == ISD::FP_TO_SINT || InOpcode == ISD::SINT_TO_FP ||
+         ((InOpcode == ISD::FP_TO_UINT || InOpcode == ISD::UINT_TO_FP) &&
+          Subtarget.hasVLX())) &&
+        (VT == MVT::v4i32 || VT == MVT::v4f32)) {
       SDValue Src = InVec.getOperand(0);
-      if (Src.getValueType().getScalarType() == MVT::f32)
+      if (Src.getValueType().getScalarSizeInBits() == 32)
         return DAG.getNode(InOpcode, DL, VT,
                            extractSubVector(Src, IdxVal, DAG, DL, SizeInBits));
     }
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index ce6b0f9c07dc8df..b0158c64c1f383c 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -2426,7 +2426,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   Ops.push_back(Callee);
 
   if (isTailCall)
-    Ops.push_back(DAG.getTargetConstant(FPDiff, dl, MVT::i32));
+    Ops.push_back(
+        DAG.getSignedConstant(FPDiff, dl, MVT::i32, /*isTarget=*/true));
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index e616a8a37c64872..a606962a5538091 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -4855,27 +4855,29 @@ multiclass avx512_binop_all<bits<8> opc, string OpcodeStr,
                             X86SchedWriteWidths sched,
                             AVX512VLVectorVTInfo _SrcVTInfo,
                             AVX512VLVectorVTInfo _DstVTInfo,
-                            SDNode OpNode, Predicate prd,  bit IsCommutable = 0> {
+                            SDNode OpNode, Predicate prd,
+                            X86VectorVTInfo _VTInfo512 = _SrcVTInfo.info512,
+                            X86VectorVTInfo _VTInfo256 = _SrcVTInfo.info256,
+                            X86VectorVTInfo _VTInfo128 = _SrcVTInfo.info128> {
   let Predicates = [prd] in
     defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, sched.ZMM, OpNode,
                                  _SrcVTInfo.info512, _DstVTInfo.info512,
-                                 v8i64_info, IsCommutable>,
-                                  EVEX_V512, EVEX_CD8<64, CD8VF>, REX_W;
+                                 _VTInfo512>, EVEX_V512;
   let Predicates = [HasVLX, prd] in {
     defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, sched.YMM, OpNode,
                                       _SrcVTInfo.info256, _DstVTInfo.info256,
-                                      v4i64x_info, IsCommutable>,
-                                      EVEX_V256, EVEX_CD8<64, CD8VF>, REX_W;
+                                      _VTInfo256>, EVEX_V256;
     defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, sched.XMM, OpNode,
                                       _SrcVTInfo.info128, _DstVTInfo.info128,
-                                      v2i64x_info, IsCommutable>,
-                                     EVEX_V128, EVEX_CD8<64, CD8VF>, REX_W;
+                                      _VTInfo128>, EVEX_V128;
   }
 }
 
 defm VPMULTISHIFTQB : avx512_binop_all<0x83, "vpmultishiftqb", SchedWriteVecALU,
-                                avx512vl_i8_info, avx512vl_i8_info,
-                                X86multishift, HasVBMI, 0>, T8;
+                                       avx512vl_i8_info, avx512vl_i8_info,
+                                       X86multishift, HasVBMI, v8i64_info,
+                                       v4i64x_info, v2i64x_info>, T8,
+                                       EVEX_CD8<64, CD8VF>, REX_W;
 
 multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
                             X86VectorVTInfo _Src, X86VectorVTInfo _Dst,
@@ -12670,34 +12672,12 @@ defm VP2INTERSECTD : avx512_vp2intersect<SchedWriteVecALU, avx512vl_i32_info>;
 defm VP2INTERSECTQ : avx512_vp2intersect<SchedWriteVecALU, avx512vl_i64_info>, REX_W;
 }
 
-multiclass avx512_binop_all2<bits<8> opc, string OpcodeStr,
-                             X86SchedWriteWidths sched,
-                             AVX512VLVectorVTInfo _SrcVTInfo,
-                             AVX512VLVectorVTInfo _DstVTInfo,
-                             SDNode OpNode, Predicate prd,
-                             bit IsCommutable = 0> {
-  let Predicates = [prd] in
-    defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, sched.ZMM, OpNode,
-                                   _SrcVTInfo.info512, _DstVTInfo.info512,
-                                   _SrcVTInfo.info512, IsCommutable>,
-                                   EVEX_V512, EVEX_CD8<32, CD8VF>;
-  let Predicates = [HasVLX, prd] in {
-    defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, sched.YMM, OpNode,
-                                      _SrcVTInfo.info256, _DstVTInfo.info256,
-                                      _SrcVTInfo.info256, IsCommutable>,
-                                     EVEX_V256, EVEX_CD8<32, CD8VF>;
-    defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, sched.XMM, OpNode,
-                                      _SrcVTInfo.info128, _DstVTInfo.info128,
-                                      _SrcVTInfo.info128, IsCommutable>,
-                                      EVEX_V128, EVEX_CD8<32, CD8VF>;
-  }
-}
-
 let ExeDomain = SSEPackedSingle in
-defm VCVTNE2PS2BF16 : avx512_binop_all2<0x72, "vcvtne2ps2bf16",
-                                        SchedWriteCvtPD2PS, //FIXME: Should be SchedWriteCvtPS2BF
-                                        avx512vl_f32_info, avx512vl_bf16_info,
-                                        X86cvtne2ps2bf16, HasBF16, 0>, T8, XD;
+defm VCVTNE2PS2BF16 : avx512_binop_all<0x72, "vcvtne2ps2bf16",
+                                       SchedWriteCvtPD2PS, //FIXME: Should be SchedWriteCvtPS2BF
+                                       avx512vl_f32_info, avx512vl_bf16_info,
+                                       X86cvtne2ps2bf16, HasBF16>, T8, XD,
+                                       EVEX_CD8<32, CD8VF>;
 
 // Truncate Float to BFloat16
 multiclass avx512_cvtps2bf16<bits<8> opc, string OpcodeStr,
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 638eb1c4f11e417..79ee9ecfdf3ce7e 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "X86RegisterInfo.h"
+#include "MCTargetDesc/X86BaseInfo.h"
 #include "X86FrameLowering.h"
 #include "X86MachineFunctionInfo.h"
 #include "X86Subtarget.h"
@@ -24,6 +25,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TileShapeInfo.h"
@@ -905,7 +907,7 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
 
   // Determine base register and offset.
-  int FIOffset;
+  int64_t FIOffset;
   Register BasePtr;
   if (MI.isReturn()) {
     assert((!hasStackRealignment(MF) ||
@@ -956,10 +958,34 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   }
 
   if (MI.getOperand(FIOperandNum+3).isImm()) {
-    // Offset is a 32-bit integer.
-    int Imm = (int)(MI.getOperand(FIOperandNum + 3).getImm());
-    int Offset = FIOffset + Imm;
-    assert((!Is64Bit || isInt<32>((long long)FIOffset + Imm)) &&
+    int64_t Imm = MI.getOperand(FIOperandNum + 3).getImm();
+    int64_t Offset = FIOffset + Imm;
+    bool FitsIn32Bits = isInt<32>(Offset);
+    // If the offset will not fit in a 32-bit displacement,
+    // then for 64-bit targets, scavenge a register to hold it.
+    // Otherwise, for 32-bit targets, this is a bug!
+    if (Is64Bit && !FitsIn32Bits) {
+      assert(RS && "RegisterScavenger was NULL");
+      const X86InstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
+      const DebugLoc &DL = MI.getDebugLoc();
+
+      RS->enterBasicBlockEnd(MBB);
+      RS->backward(std::next(II));
+
+      Register ScratchReg = RS->scavengeRegisterBackwards(
+          X86::GR64RegClass, II, /*RestoreAfter=*/false, /*SPAdj=*/0,
+          /*AllowSpill=*/true);
+      assert(ScratchReg != 0 && "scratch reg was 0");
+      RS->setRegUsed(ScratchReg);
+
+      BuildMI(MBB, II, DL, TII->get(X86::MOV64ri), ScratchReg).addImm(Offset);
+
+      MI.getOperand(FIOperandNum + 3).setImm(0);
+      MI.getOperand(FIOperandNum + 2).setReg(ScratchReg);
+
+      return false;
+    }
+    assert((Is64Bit || FitsIn32Bits) &&
            "Requesting 64-bit offset in 32-bit immediate!");
     if (Offset != 0 || !tryOptimizeLEAtoMOV(II))
       MI.getOperand(FIOperandNum + 3).ChangeToImmediate(Offset);
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.h b/llvm/lib/Target/X86/X86RegisterInfo.h
index 7296a5f021e4ad4..dd03a108fb8e693 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.h
+++ b/llvm/lib/Target/X86/X86RegisterInfo.h
@@ -13,6 +13,8 @@
 #ifndef LLVM_LIB_TARGET_X86_X86REGISTERINFO_H
 #define LLVM_LIB_TARGET_X86_X86REGISTERINFO_H
 
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 
 #define GET_REGINFO_HEADER
@@ -176,6 +178,13 @@ class X86RegisterInfo final : public X86GenRegisterInfo {
                              SmallVectorImpl<MCPhysReg> &Hints,
                              const MachineFunction &MF, const VirtRegMap *VRM,
                              const LiveRegMatrix *Matrix) const override;
+
+  bool requiresRegisterScavenging(const MachineFunction &MF) const override {
+    const MachineFrameInfo &MFI = MF.getFrameInfo();
+
+    // We need to register scavenge if the frame is very large.
+    return !isInt<32>(MFI.estimateStackSize(MF));
+  }
 };
 
 } // End llvm namespace
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver1.td b/llvm/lib/Target/X86/X86ScheduleZnver1.td
index 7ee9eadf8439085..a044ddc30013295 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver1.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver1.td
@@ -986,10 +986,21 @@ def : InstRW<[ZnWritePCMPGTQYm], (instrs VPCMPGTQYrm)>;
 //-- Move instructions --//
 
 // VPERM2F128 / VPERM2I128.
-def : InstRW<[WriteMicrocoded], (instrs VPERM2F128rr,
-                                        VPERM2I128rr)>;
-def : InstRW<[WriteMicrocoded], (instrs VPERM2F128rm,
-                                        VPERM2I128rm)>;
+def ZnWriteVPERM2r : SchedWriteRes<[ZnFPU0, ZnFPU12]> {
+  let NumMicroOps = 8;
+  let Latency = 3;
+  let ReleaseAtCycles = [3,3];
+}
+def : InstRW<[ZnWriteVPERM2r], (instrs VPERM2F128rr,
+                                       VPERM2I128rr)>;
+
+def ZnWriteVPERM2m : SchedWriteRes<[ZnAGU, ZnFPU0, ZnFPU12]> {
+  let NumMicroOps = 12;
+  let Latency = 8;
+  let ReleaseAtCycles = [1,3,3];
+}
+def : InstRW<[ZnWriteVPERM2m], (instrs VPERM2F128rm,
+                                       VPERM2I128rm)>;
 
 def ZnWriteBROADCAST : SchedWriteRes<[ZnAGU, ZnFPU13]> {
   let NumMicroOps = 2;
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver2.td b/llvm/lib/Target/X86/X86ScheduleZnver2.td
index c0775847798d27d..c3a0f2684a29898 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver2.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver2.td
@@ -994,9 +994,18 @@ def : InstRW<[Zn2WritePCMPGTQYm], (instrs VPCMPGTQYrm)>;
 //-- Move instructions --//
 
 // VPERM2F128 / VPERM2I128.
-def : InstRW<[WriteMicrocoded], (instrs VPERM2F128rr,
+def Zn2WriteVPERM2r : SchedWriteRes<[Zn2FPU2]> {
+  let NumMicroOps = 1;
+  let Latency = 3;
+}
+def : InstRW<[Zn2WriteVPERM2r], (instrs VPERM2F128rr,
                                         VPERM2I128rr)>;
-def : InstRW<[WriteMicrocoded], (instrs VPERM2F128rm,
+
+def Zn2WriteVPERM2m : SchedWriteRes<[Zn2AGU, Zn2FPU2]> {
+  let NumMicroOps = 1;
+  let Latency = 8;
+}
+def : InstRW<[Zn2WriteVPERM2m], (instrs VPERM2F128rm,
                                         VPERM2I128rm)>;
 
 def Zn2WriteBROADCAST : SchedWriteRes<[Zn2AGU, Zn2FPU13]> {
@@ -1011,7 +1020,7 @@ def : InstRW<[Zn2WriteBROADCAST], (instrs VBROADCASTF128rm,
 // r32,x,i.
 def Zn2WriteEXTRACTPSr : SchedWriteRes<[Zn2FPU12, Zn2FPU2]> {
   let Latency = 2;
-  let NumMicroOps = 2; 
+  let NumMicroOps = 2;
   let ReleaseAtCycles = [1, 2];
 }
 def : InstRW<[Zn2WriteEXTRACTPSr], (instregex "(V?)EXTRACTPSrr")>;
diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 59df02daf632652..d82501ebf8d2fb8 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -721,6 +721,16 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   return RISCVISAInfo::postProcessAndChecking(std::move(ISAInfo));
 }
 
+static Error getIncompatibleError(StringRef Ext1, StringRef Ext2) {
+  return getError("'" + Ext1 + "' and '" + Ext2 +
+                  "' extensions are incompatible");
+}
+
+static Error getExtensionRequiresError(StringRef Ext, StringRef ReqExt) {
+  return getError("'" + Ext + "' requires '" + ReqExt +
+                  "' extension to also be specified");
+}
+
 Error RISCVISAInfo::checkDependency() {
   bool HasE = Exts.count("e") != 0;
   bool HasI = Exts.count("i") != 0;
@@ -733,32 +743,24 @@ Error RISCVISAInfo::checkDependency() {
   bool HasZcmt = Exts.count("zcmt") != 0;
 
   if (HasI && HasE)
-    return getError("'I' and 'E' extensions are incompatible");
+    return getIncompatibleError("i", "e");
 
   if (HasF && HasZfinx)
-    return getError("'f' and 'zfinx' extensions are incompatible");
+    return getIncompatibleError("f", "zfinx");
 
   if (HasZvl && !HasVector)
-    return getError(
-        "'zvl*b' requires 'v' or 'zve*' extension to also be specified");
-
-  if (Exts.count("zvbb") && !HasVector)
-    return getError(
-        "'zvbb' requires 'v' or 'zve*' extension to also be specified");
+    return getExtensionRequiresError("zvl*b", "v' or 'zve*");
 
-  if (Exts.count("zvbc") && !Exts.count("zve64x"))
-    return getError(
-        "'zvbc' requires 'v' or 'zve64*' extension to also be specified");
-
-  if ((Exts.count("zvkb") || Exts.count("zvkg") || Exts.count("zvkned") ||
-       Exts.count("zvknha") || Exts.count("zvksed") || Exts.count("zvksh")) &&
-      !HasVector)
-    return getError(
-        "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+  if (!HasVector)
+    for (auto Ext :
+         {"zvbb", "zvbc32e", "zvkb", "zvkg", "zvkgs", "zvkned", "zvknha", "zvksed", "zvksh"})
+      if (Exts.count(Ext))
+        return getExtensionRequiresError(Ext, "v' or 'zve*");
 
-  if (Exts.count("zvknhb") && !Exts.count("zve64x"))
-    return getError(
-        "'zvknhb' requires 'v' or 'zve64*' extension to also be specified");
+  if (!Exts.count("zve64x"))
+    for (auto Ext : {"zvknhb", "zvbc"})
+      if (Exts.count(Ext))
+        return getExtensionRequiresError(Ext, "v' or 'zve64*");
 
   if ((HasZcmt || Exts.count("zcmp")) && HasD && (HasC || Exts.count("zcd")))
     return getError(Twine("'") + (HasZcmt ? "zcmt" : "zcmp") +
@@ -769,23 +771,20 @@ Error RISCVISAInfo::checkDependency() {
   if (XLen != 32 && Exts.count("zcf"))
     return getError("'zcf' is only supported for 'rv32'");
 
-  if (Exts.count("zacas") && !(Exts.count("a") || Exts.count("zaamo")))
-    return getError(
-        "'zacas' requires 'a' or 'zaamo' extension to also be specified");
-
-  if (Exts.count("zabha") && !(Exts.count("a") || Exts.count("zaamo")))
-    return getError(
-        "'zabha' requires 'a' or 'zaamo' extension to also be specified");
+  if (!(Exts.count("a") || Exts.count("zaamo")))
+    for (auto Ext : {"zacas", "zabha"})
+      if (Exts.count(Ext))
+        return getExtensionRequiresError(Ext, "a' or 'zaamo");
 
   if (Exts.count("xwchc") != 0) {
     if (XLen != 32)
-      return getError("'Xwchc' is only supported for 'rv32'");
+      return getError("'xwchc' is only supported for 'rv32'");
 
     if (HasD)
-      return getError("'D' and 'Xwchc' extensions are incompatible");
+      return getIncompatibleError("d", "xwchc");
 
     if (Exts.count("zcb") != 0)
-      return getError("'Xwchc' and 'Zcb' extensions are incompatible");
+      return getIncompatibleError("xwchc", "zcb");
   }
 
   return Error::success();
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 41a7fc0870cf0ad..9f033028347f0d2 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1628,6 +1628,8 @@ ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
                       << "\n");
     assert(OffsetInfoMap.count(CurPtr) &&
            "The current pointer offset should have been seeded!");
+    assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
+           "Current pointer should be assigned");
 
     if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
       if (CE->isCast())
@@ -1906,6 +1908,7 @@ ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
   };
   auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
     assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
+    assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
     if (OffsetInfoMap.count(NewU)) {
       LLVM_DEBUG({
         if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
@@ -1916,8 +1919,8 @@ ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
       });
       return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
     }
-    OffsetInfoMap[NewU] = OffsetInfoMap[OldU];
-    return true;
+    bool Unused;
+    return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
   };
   if (!A.checkForAllUses(UsePred, *this, AssociatedValue,
                          /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL,
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 798bcb77328fc81..baf39c908706c6e 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -5569,6 +5569,8 @@ void OpenMPOpt::registerAAsForFunction(Attributor &A, const Function &F) {
       bool UsedAssumedInformation = false;
       A.getAssumedSimplified(IRPosition::value(*LI), /* AA */ nullptr,
                              UsedAssumedInformation, AA::Interprocedural);
+      A.getOrCreateAAFor<AAAddressSpace>(
+          IRPosition::value(*LI->getPointerOperand()));
       continue;
     }
     if (auto *CI = dyn_cast<CallBase>(&I)) {
@@ -5578,6 +5580,8 @@ void OpenMPOpt::registerAAsForFunction(Attributor &A, const Function &F) {
     }
     if (auto *SI = dyn_cast<StoreInst>(&I)) {
       A.getOrCreateAAFor<AAIsDead>(IRPosition::value(*SI));
+      A.getOrCreateAAFor<AAAddressSpace>(
+          IRPosition::value(*SI->getPointerOperand()));
       continue;
     }
     if (auto *FI = dyn_cast<FenceInst>(&I)) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 3bd086230cbec5a..123f810bacfb6e4 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1548,8 +1548,12 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
   }
 
   // A + -B  -->  A - B
-  if (match(RHS, m_Neg(m_Value(B))))
-    return BinaryOperator::CreateSub(LHS, B);
+  if (match(RHS, m_Neg(m_Value(B)))) {
+    auto *Sub = BinaryOperator::CreateSub(LHS, B);
+    auto *OBO = cast<OverflowingBinaryOperator>(RHS);
+    Sub->setHasNoSignedWrap(I.hasNoSignedWrap() && OBO->hasNoSignedWrap());
+    return Sub;
+  }
 
   if (Value *V = checkForNegativeOperand(I, Builder))
     return replaceInstUsesWith(I, V);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 4f9a5bd2c17f037..2c2e1bc4686a4ed 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3692,7 +3692,7 @@ Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {
 
   // Skip optimizing notail and musttail calls so
   // LibCallSimplifier::optimizeCall doesn't have to preserve those invariants.
-  // LibCallSimplifier::optimizeCall should try to preseve tail calls though.
+  // LibCallSimplifier::optimizeCall should try to preserve tail calls though.
   if (CI->isMustTailCall() || CI->isNoTailCall())
     return nullptr;
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 10488ecb747a484..34c9e0fde4f4285 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -4018,6 +4018,16 @@ foldICmpOfCmpIntrinsicWithConstant(ICmpInst::Predicate Pred, IntrinsicInst *I,
       NewPredicate = ICmpInst::ICMP_ULE;
     break;
 
+  case ICmpInst::ICMP_ULT:
+    if (C.ugt(1))
+      NewPredicate = ICmpInst::ICMP_UGE;
+    break;
+
+  case ICmpInst::ICMP_UGT:
+    if (!C.isZero() && !C.isAllOnes())
+      NewPredicate = ICmpInst::ICMP_ULT;
+    break;
+
   default:
     break;
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 16f1c3ba15eba2d..a0e846c3b5a566b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -729,6 +729,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
 
   // Helpers of visitSelectInst().
   Instruction *foldSelectOfBools(SelectInst &SI);
+  Instruction *foldSelectToCmp(SelectInst &SI);
   Instruction *foldSelectExtConst(SelectInst &Sel);
   Instruction *foldSelectOpOp(SelectInst &SI, Instruction *TI, Instruction *FI);
   Instruction *foldSelectIntoOp(SelectInst &SI, Value *, Value *);
@@ -780,11 +781,14 @@ class Negator final {
   using BuilderTy = IRBuilder<TargetFolder, IRBuilderCallbackInserter>;
   BuilderTy Builder;
 
+  const DominatorTree &DT;
+
   const bool IsTrulyNegation;
 
   SmallDenseMap<Value *, Value *> NegationsCache;
 
-  Negator(LLVMContext &C, const DataLayout &DL, bool IsTrulyNegation);
+  Negator(LLVMContext &C, const DataLayout &DL, const DominatorTree &DT,
+          bool IsTrulyNegation);
 
 #if LLVM_ENABLE_STATS
   unsigned NumValuesVisitedInThisNegator = 0;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
index 2dd1db6a4a757bb..92293ef401465b7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
@@ -90,13 +90,14 @@ static cl::opt<unsigned>
                     cl::desc("What is the maximal lookup depth when trying to "
                              "check for viability of negation sinking."));
 
-Negator::Negator(LLVMContext &C, const DataLayout &DL, bool IsTrulyNegation_)
+Negator::Negator(LLVMContext &C, const DataLayout &DL, const DominatorTree &DT_,
+                 bool IsTrulyNegation_)
     : Builder(C, TargetFolder(DL),
               IRBuilderCallbackInserter([&](Instruction *I) {
                 ++NegatorNumInstructionsCreatedTotal;
                 NewInstructions.push_back(I);
               })),
-      IsTrulyNegation(IsTrulyNegation_) {}
+      DT(DT_), IsTrulyNegation(IsTrulyNegation_) {}
 
 #if LLVM_ENABLE_STATS
 Negator::~Negator() {
@@ -309,6 +310,9 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
     auto *PHI = cast<PHINode>(I);
     SmallVector<Value *, 4> NegatedIncomingValues(PHI->getNumOperands());
     for (auto I : zip(PHI->incoming_values(), NegatedIncomingValues)) {
+      // Don't negate indvars to avoid infinite loops.
+      if (DT.dominates(PHI->getParent(), std::get<0>(I)))
+        return nullptr;
       if (!(std::get<1>(I) =
                 negate(std::get<0>(I), IsNSW, Depth + 1))) // Early return.
         return nullptr;
@@ -537,7 +541,8 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
   if (!NegatorEnabled || !DebugCounter::shouldExecute(NegatorCounter))
     return nullptr;
 
-  Negator N(Root->getContext(), IC.getDataLayout(), LHSIsZero);
+  Negator N(Root->getContext(), IC.getDataLayout(), IC.getDominatorTree(),
+            LHSIsZero);
   std::optional<Result> Res = N.run(Root, IsNSW);
   if (!Res) { // Negation failed.
     LLVM_DEBUG(dbgs() << "Negator: failed to sink negation into " << *Root
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index b86f27168303a1e..1f6d5759883fd0c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -679,11 +679,11 @@ static Value *foldSelectICmpLshrAshr(const ICmpInst *IC, Value *TrueVal,
   Value *X, *Y;
   unsigned Bitwidth = CmpRHS->getType()->getScalarSizeInBits();
   if ((Pred != ICmpInst::ICMP_SGT ||
-       !match(CmpRHS,
-              m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, -1)))) &&
+       !match(CmpRHS, m_SpecificInt_ICMP(ICmpInst::ICMP_SGE,
+                                         APInt::getAllOnes(Bitwidth)))) &&
       (Pred != ICmpInst::ICMP_SLT ||
-       !match(CmpRHS,
-              m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, 0)))))
+       !match(CmpRHS, m_SpecificInt_ICMP(ICmpInst::ICMP_SGE,
+                                         APInt::getZero(Bitwidth)))))
     return nullptr;
 
   // Canonicalize so that ashr is in FalseVal.
@@ -3558,6 +3558,55 @@ static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder) {
                                 Masked);
 }
 
+// This function tries to fold the following operations:
+//   (x < y) ? -1 : zext(x != y)
+//   (x > y) ? 1 : sext(x != y)
+// Into ucmp/scmp(x, y), where signedness is determined by the signedness
+// of the comparison in the original sequence.
+Instruction *InstCombinerImpl::foldSelectToCmp(SelectInst &SI) {
+  Value *TV = SI.getTrueValue();
+  Value *FV = SI.getFalseValue();
+
+  ICmpInst::Predicate Pred;
+  Value *LHS, *RHS;
+  if (!match(SI.getCondition(), m_ICmp(Pred, m_Value(LHS), m_Value(RHS))))
+    return nullptr;
+
+  if (!LHS->getType()->isIntOrIntVectorTy())
+    return nullptr;
+
+  // Try to swap operands and the predicate. We need to be careful when doing
+  // so because two of the patterns have opposite predicates, so use the
+  // constant inside select to determine if swapping operands would be
+  // beneficial to us.
+  if ((ICmpInst::isGT(Pred) && match(TV, m_AllOnes())) ||
+      (ICmpInst::isLT(Pred) && match(TV, m_One()))) {
+    Pred = ICmpInst::getSwappedPredicate(Pred);
+    std::swap(LHS, RHS);
+  }
+
+  Intrinsic::ID IID =
+      ICmpInst::isSigned(Pred) ? Intrinsic::scmp : Intrinsic::ucmp;
+
+  bool Replace = false;
+  // (x < y) ? -1 : zext(x != y)
+  if (ICmpInst::isLT(Pred) && match(TV, m_AllOnes()) &&
+      match(FV, m_ZExt(m_c_SpecificICmp(ICmpInst::ICMP_NE, m_Specific(LHS),
+                                        m_Specific(RHS)))))
+    Replace = true;
+
+  // (x > y) ? 1 : sext(x != y)
+  if (ICmpInst::isGT(Pred) && match(TV, m_One()) &&
+      match(FV, m_SExt(m_c_SpecificICmp(ICmpInst::ICMP_NE, m_Specific(LHS),
+                                        m_Specific(RHS)))))
+    Replace = true;
+
+  if (Replace)
+    return replaceInstUsesWith(
+        SI, Builder.CreateIntrinsic(SI.getType(), IID, {LHS, RHS}));
+  return nullptr;
+}
+
 bool InstCombinerImpl::fmulByZeroIsZero(Value *MulVal, FastMathFlags FMF,
                                         const Instruction *CtxI) const {
   KnownFPClass Known = computeKnownFPClass(MulVal, FMF, fcNegative, CtxI);
@@ -4061,6 +4110,9 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
   if (Instruction *I = foldBitCeil(SI, Builder))
     return I;
 
+  if (Instruction *I = foldSelectToCmp(SI))
+    return I;
+
   // Fold:
   // (select A && B, T, F) -> (select A, (select B, T, F), F)
   // (select A || B, T, F) -> (select A, T, (select B, T, F))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 38f8a41214b6825..794b384d126eb66 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -511,6 +511,21 @@ Instruction *InstCombinerImpl::commonShiftTransforms(BinaryOperator &I) {
   if (match(Op1, m_Or(m_Value(), m_SpecificInt(BitWidth - 1))))
     return replaceOperand(I, 1, ConstantInt::get(Ty, BitWidth - 1));
 
+  Instruction *CmpIntr;
+  if ((I.getOpcode() == Instruction::LShr ||
+       I.getOpcode() == Instruction::AShr) &&
+      match(Op0, m_OneUse(m_Instruction(CmpIntr))) &&
+      isa<CmpIntrinsic>(CmpIntr) &&
+      match(Op1, m_SpecificInt(Ty->getScalarSizeInBits() - 1))) {
+    Value *Cmp =
+        Builder.CreateICmp(cast<CmpIntrinsic>(CmpIntr)->getLTPredicate(),
+                           CmpIntr->getOperand(0), CmpIntr->getOperand(1));
+    return CastInst::Create(I.getOpcode() == Instruction::LShr
+                                ? Instruction::ZExt
+                                : Instruction::SExt,
+                            Cmp, Ty);
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index d1bb1334aae6a36..5de0a78d087968b 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -642,6 +642,10 @@ static uint64_t GetCtorAndDtorPriority(Triple &TargetTriple) {
   }
 }
 
+static Twine genName(StringRef suffix) {
+  return Twine(kAsanGenPrefix) + suffix;
+}
+
 namespace {
 /// Helper RAII class to post-process inserted asan runtime calls during a
 /// pass on a single Function. Upon end of scope, detects and applies the
@@ -719,7 +723,8 @@ struct AddressSanitizer {
                    bool Recover = false, bool UseAfterScope = false,
                    AsanDetectStackUseAfterReturnMode UseAfterReturn =
                        AsanDetectStackUseAfterReturnMode::Runtime)
-      : CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
+      : M(M),
+        CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
                                                             : CompileKernel),
         Recover(ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover),
         UseAfterScope(UseAfterScope || ClUseAfterScope),
@@ -803,7 +808,7 @@ struct AddressSanitizer {
 private:
   friend struct FunctionStackPoisoner;
 
-  void initializeCallbacks(Module &M, const TargetLibraryInfo *TLI);
+  void initializeCallbacks(const TargetLibraryInfo *TLI);
 
   bool LooksLikeCodeInBug11395(Instruction *I);
   bool GlobalIsLinkerInitialized(GlobalVariable *G);
@@ -826,6 +831,7 @@ struct AddressSanitizer {
     }
   };
 
+  Module &M;
   LLVMContext *C;
   const DataLayout *DL;
   Triple TargetTriple;
@@ -868,7 +874,8 @@ class ModuleAddressSanitizer {
                          bool UseGlobalsGC = true, bool UseOdrIndicator = true,
                          AsanDtorKind DestructorKind = AsanDtorKind::Global,
                          AsanCtorKind ConstructorKind = AsanCtorKind::Global)
-      : CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
+      : M(M),
+        CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
                                                             : CompileKernel),
         InsertVersionCheck(ClInsertVersionCheck.getNumOccurrences() > 0
                                ? ClInsertVersionCheck
@@ -906,45 +913,47 @@ class ModuleAddressSanitizer {
     assert(this->DestructorKind != AsanDtorKind::Invalid);
   }
 
-  bool instrumentModule(Module &);
+  bool instrumentModule();
 
 private:
-  void initializeCallbacks(Module &M);
+  void initializeCallbacks();
 
-  void instrumentGlobals(IRBuilder<> &IRB, Module &M, bool *CtorComdat);
-  void InstrumentGlobalsCOFF(IRBuilder<> &IRB, Module &M,
+  void instrumentGlobals(IRBuilder<> &IRB, bool *CtorComdat);
+  void InstrumentGlobalsCOFF(IRBuilder<> &IRB,
                              ArrayRef<GlobalVariable *> ExtendedGlobals,
                              ArrayRef<Constant *> MetadataInitializers);
-  void instrumentGlobalsELF(IRBuilder<> &IRB, Module &M,
+  void instrumentGlobalsELF(IRBuilder<> &IRB,
                             ArrayRef<GlobalVariable *> ExtendedGlobals,
                             ArrayRef<Constant *> MetadataInitializers,
                             const std::string &UniqueModuleId);
-  void InstrumentGlobalsMachO(IRBuilder<> &IRB, Module &M,
+  void InstrumentGlobalsMachO(IRBuilder<> &IRB,
                               ArrayRef<GlobalVariable *> ExtendedGlobals,
                               ArrayRef<Constant *> MetadataInitializers);
   void
-  InstrumentGlobalsWithMetadataArray(IRBuilder<> &IRB, Module &M,
+  InstrumentGlobalsWithMetadataArray(IRBuilder<> &IRB,
                                      ArrayRef<GlobalVariable *> ExtendedGlobals,
                                      ArrayRef<Constant *> MetadataInitializers);
 
-  GlobalVariable *CreateMetadataGlobal(Module &M, Constant *Initializer,
+  GlobalVariable *CreateMetadataGlobal(Constant *Initializer,
                                        StringRef OriginalName);
   void SetComdatForGlobalMetadata(GlobalVariable *G, GlobalVariable *Metadata,
                                   StringRef InternalSuffix);
-  Instruction *CreateAsanModuleDtor(Module &M);
+  Instruction *CreateAsanModuleDtor();
 
   const GlobalVariable *getExcludedAliasedGlobal(const GlobalAlias &GA) const;
   bool shouldInstrumentGlobal(GlobalVariable *G) const;
   bool ShouldUseMachOGlobalsSection() const;
   StringRef getGlobalMetadataSection() const;
-  void poisonOneInitializer(Function &GlobalInit, GlobalValue *ModuleName);
-  void createInitializerPoisonCalls(Module &M, GlobalValue *ModuleName);
+  void poisonOneInitializer(Function &GlobalInit);
+  void createInitializerPoisonCalls();
   uint64_t getMinRedzoneSizeForGlobal() const {
     return getRedzoneSizeForScale(Mapping.Scale);
   }
   uint64_t getRedzoneSizeForGlobal(uint64_t SizeInBytes) const;
-  int GetAsanVersion(const Module &M) const;
+  int GetAsanVersion() const;
+  GlobalVariable *getOrCreateModuleName();
 
+  Module &M;
   bool CompileKernel;
   bool InsertVersionCheck;
   bool Recover;
@@ -970,6 +979,7 @@ class ModuleAddressSanitizer {
 
   Function *AsanCtorFunction = nullptr;
   Function *AsanDtorFunction = nullptr;
+  GlobalVariable *ModuleName = nullptr;
 };
 
 // Stack poisoning does not play well with exception handling.
@@ -1271,7 +1281,7 @@ PreservedAnalyses AddressSanitizerPass::run(Module &M,
     const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
     Modified |= FunctionSanitizer.instrumentFunction(F, &TLI);
   }
-  Modified |= ModuleSanitizer.instrumentModule(M);
+  Modified |= ModuleSanitizer.instrumentModule();
   if (!Modified)
     return PreservedAnalyses::all();
 
@@ -1957,14 +1967,14 @@ void AddressSanitizer::instrumentUnusualSizeOrAlignment(
   }
 }
 
-void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit,
-                                                  GlobalValue *ModuleName) {
+void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit) {
   // Set up the arguments to our poison/unpoison functions.
   IRBuilder<> IRB(&GlobalInit.front(),
                   GlobalInit.front().getFirstInsertionPt());
 
   // Add a call to poison all external globals before the given function starts.
-  Value *ModuleNameAddr = ConstantExpr::getPointerCast(ModuleName, IntptrTy);
+  Value *ModuleNameAddr =
+      ConstantExpr::getPointerCast(getOrCreateModuleName(), IntptrTy);
   IRB.CreateCall(AsanPoisonGlobals, ModuleNameAddr);
 
   // Add calls to unpoison all globals before each return instruction.
@@ -1973,8 +1983,7 @@ void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit,
       CallInst::Create(AsanUnpoisonGlobals, "", RI->getIterator());
 }
 
-void ModuleAddressSanitizer::createInitializerPoisonCalls(
-    Module &M, GlobalValue *ModuleName) {
+void ModuleAddressSanitizer::createInitializerPoisonCalls() {
   GlobalVariable *GV = M.getGlobalVariable("llvm.global_ctors");
   if (!GV)
     return;
@@ -1994,7 +2003,7 @@ void ModuleAddressSanitizer::createInitializerPoisonCalls(
       // Don't instrument CTORs that will run before asan.module_ctor.
       if (Priority->getLimitedValue() <= GetCtorAndDtorPriority(TargetTriple))
         continue;
-      poisonOneInitializer(*F, ModuleName);
+      poisonOneInitializer(*F);
     }
   }
 }
@@ -2198,7 +2207,7 @@ StringRef ModuleAddressSanitizer::getGlobalMetadataSection() const {
   llvm_unreachable("unsupported object format");
 }
 
-void ModuleAddressSanitizer::initializeCallbacks(Module &M) {
+void ModuleAddressSanitizer::initializeCallbacks() {
   IRBuilder<> IRB(*C);
 
   // Declare our poisoning and unpoisoning functions.
@@ -2239,7 +2248,7 @@ void ModuleAddressSanitizer::SetComdatForGlobalMetadata(
       // If G is unnamed, it must be internal. Give it an artificial name
       // so we can put it in a comdat.
       assert(G->hasLocalLinkage());
-      G->setName(Twine(kAsanGenPrefix) + "_anon_global");
+      G->setName(genName("anon_global"));
     }
 
     if (!InternalSuffix.empty() && G->hasLocalLinkage()) {
@@ -2268,7 +2277,7 @@ void ModuleAddressSanitizer::SetComdatForGlobalMetadata(
 // Create a separate metadata global and put it in the appropriate ASan
 // global registration section.
 GlobalVariable *
-ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer,
+ModuleAddressSanitizer::CreateMetadataGlobal(Constant *Initializer,
                                              StringRef OriginalName) {
   auto Linkage = TargetTriple.isOSBinFormatMachO()
                      ? GlobalVariable::InternalLinkage
@@ -2283,7 +2292,7 @@ ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer,
   return Metadata;
 }
 
-Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor(Module &M) {
+Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor() {
   AsanDtorFunction = Function::createWithDefaultAttr(
       FunctionType::get(Type::getVoidTy(*C), false),
       GlobalValue::InternalLinkage, 0, kAsanModuleDtorName, &M);
@@ -2296,7 +2305,7 @@ Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor(Module &M) {
 }
 
 void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
-    IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
+    IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers) {
   assert(ExtendedGlobals.size() == MetadataInitializers.size());
   auto &DL = M.getDataLayout();
@@ -2305,8 +2314,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
     Constant *Initializer = MetadataInitializers[i];
     GlobalVariable *G = ExtendedGlobals[i];
-    GlobalVariable *Metadata =
-        CreateMetadataGlobal(M, Initializer, G->getName());
+    GlobalVariable *Metadata = CreateMetadataGlobal(Initializer, G->getName());
     MDNode *MD = MDNode::get(M.getContext(), ValueAsMetadata::get(G));
     Metadata->setMetadata(LLVMContext::MD_associated, MD);
     MetadataGlobals[i] = Metadata;
@@ -2329,7 +2337,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
 }
 
 void ModuleAddressSanitizer::instrumentGlobalsELF(
-    IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
+    IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers,
     const std::string &UniqueModuleId) {
   assert(ExtendedGlobals.size() == MetadataInitializers.size());
@@ -2344,7 +2352,7 @@ void ModuleAddressSanitizer::instrumentGlobalsELF(
   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
     GlobalVariable *G = ExtendedGlobals[i];
     GlobalVariable *Metadata =
-        CreateMetadataGlobal(M, MetadataInitializers[i], G->getName());
+        CreateMetadataGlobal(MetadataInitializers[i], G->getName());
     MDNode *MD = MDNode::get(M.getContext(), ValueAsMetadata::get(G));
     Metadata->setMetadata(LLVMContext::MD_associated, MD);
     MetadataGlobals[i] = Metadata;
@@ -2389,7 +2397,7 @@ void ModuleAddressSanitizer::instrumentGlobalsELF(
   // We also need to unregister globals at the end, e.g., when a shared library
   // gets closed.
   if (DestructorKind != AsanDtorKind::None && !MetadataGlobals.empty()) {
-    IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
+    IRBuilder<> IrbDtor(CreateAsanModuleDtor());
     IrbDtor.CreateCall(AsanUnregisterElfGlobals,
                        {IRB.CreatePointerCast(RegisteredFlag, IntptrTy),
                         IRB.CreatePointerCast(StartELFMetadata, IntptrTy),
@@ -2398,7 +2406,7 @@ void ModuleAddressSanitizer::instrumentGlobalsELF(
 }
 
 void ModuleAddressSanitizer::InstrumentGlobalsMachO(
-    IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
+    IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers) {
   assert(ExtendedGlobals.size() == MetadataInitializers.size());
 
@@ -2411,8 +2419,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsMachO(
   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
     Constant *Initializer = MetadataInitializers[i];
     GlobalVariable *G = ExtendedGlobals[i];
-    GlobalVariable *Metadata =
-        CreateMetadataGlobal(M, Initializer, G->getName());
+    GlobalVariable *Metadata = CreateMetadataGlobal(Initializer, G->getName());
 
     // On recent Mach-O platforms, we emit the global metadata in a way that
     // allows the linker to properly strip dead globals.
@@ -2451,14 +2458,14 @@ void ModuleAddressSanitizer::InstrumentGlobalsMachO(
   // We also need to unregister globals at the end, e.g., when a shared library
   // gets closed.
   if (DestructorKind != AsanDtorKind::None) {
-    IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
+    IRBuilder<> IrbDtor(CreateAsanModuleDtor());
     IrbDtor.CreateCall(AsanUnregisterImageGlobals,
                        {IRB.CreatePointerCast(RegisteredFlag, IntptrTy)});
   }
 }
 
 void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
-    IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
+    IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers) {
   assert(ExtendedGlobals.size() == MetadataInitializers.size());
   unsigned N = ExtendedGlobals.size();
@@ -2482,7 +2489,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
   // We also need to unregister globals at the end, e.g., when a shared library
   // gets closed.
   if (DestructorKind != AsanDtorKind::None) {
-    IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
+    IRBuilder<> IrbDtor(CreateAsanModuleDtor());
     IrbDtor.CreateCall(AsanUnregisterGlobals,
                        {IRB.CreatePointerCast(AllGlobals, IntptrTy),
                         ConstantInt::get(IntptrTy, N)});
@@ -2494,7 +2501,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
 // redzones and inserts this function into llvm.global_ctors.
 // Sets *CtorComdat to true if the global registration code emitted into the
 // asan constructor is comdat-compatible.
-void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
+void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
                                                bool *CtorComdat) {
   // Build set of globals that are aliased by some GA, where
   // getExcludedAliasedGlobal(GA) returns the relevant GlobalVariable.
@@ -2531,16 +2538,6 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
   SmallVector<GlobalVariable *, 16> NewGlobals(n);
   SmallVector<Constant *, 16> Initializers(n);
 
-  bool HasDynamicallyInitializedGlobals = false;
-
-  // We shouldn't merge same module names, as this string serves as unique
-  // module ID in runtime.
-  GlobalVariable *ModuleName =
-      n != 0
-          ? createPrivateGlobalForString(M, M.getModuleIdentifier(),
-                                         /*AllowMerging*/ false, kAsanGenPrefix)
-          : nullptr;
-
   for (size_t i = 0; i < n; i++) {
     GlobalVariable *G = GlobalsToChange[i];
 
@@ -2554,7 +2551,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
     std::string NameForGlobal = G->getName().str();
     GlobalVariable *Name =
         createPrivateGlobalForString(M, llvm::demangle(NameForGlobal),
-                                     /*AllowMerging*/ true, kAsanGenPrefix);
+                                     /*AllowMerging*/ true, genName("global"));
 
     Type *Ty = G->getValueType();
     const uint64_t SizeInBytes = DL.getTypeAllocSize(Ty);
@@ -2641,14 +2638,11 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
         ConstantInt::get(IntptrTy, SizeInBytes),
         ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize),
         ConstantExpr::getPointerCast(Name, IntptrTy),
-        ConstantExpr::getPointerCast(ModuleName, IntptrTy),
+        ConstantExpr::getPointerCast(getOrCreateModuleName(), IntptrTy),
         ConstantInt::get(IntptrTy, MD.IsDynInit),
         Constant::getNullValue(IntptrTy),
         ConstantExpr::getPointerCast(ODRIndicator, IntptrTy));
 
-    if (ClInitializers && MD.IsDynInit)
-      HasDynamicallyInitializedGlobals = true;
-
     LLVM_DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
 
     Initializers[i] = Initializer;
@@ -2670,8 +2664,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
     // function will be called. The module destructor is not created when n ==
     // 0.
     *CtorComdat = true;
-    instrumentGlobalsELF(IRB, M, NewGlobals, Initializers,
-                         getUniqueModuleId(&M));
+    instrumentGlobalsELF(IRB, NewGlobals, Initializers, getUniqueModuleId(&M));
   } else if (n == 0) {
     // When UseGlobalsGC is false, COMDAT can still be used if n == 0, because
     // all compile units will have identical module constructor/destructor.
@@ -2679,17 +2672,17 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
   } else {
     *CtorComdat = false;
     if (UseGlobalsGC && TargetTriple.isOSBinFormatCOFF()) {
-      InstrumentGlobalsCOFF(IRB, M, NewGlobals, Initializers);
+      InstrumentGlobalsCOFF(IRB, NewGlobals, Initializers);
     } else if (UseGlobalsGC && ShouldUseMachOGlobalsSection()) {
-      InstrumentGlobalsMachO(IRB, M, NewGlobals, Initializers);
+      InstrumentGlobalsMachO(IRB, NewGlobals, Initializers);
     } else {
-      InstrumentGlobalsWithMetadataArray(IRB, M, NewGlobals, Initializers);
+      InstrumentGlobalsWithMetadataArray(IRB, NewGlobals, Initializers);
     }
   }
 
   // Create calls for poisoning before initializers run and unpoisoning after.
-  if (HasDynamicallyInitializedGlobals)
-    createInitializerPoisonCalls(M, ModuleName);
+  if (ClInitializers)
+    createInitializerPoisonCalls();
 
   LLVM_DEBUG(dbgs() << M);
 }
@@ -2719,7 +2712,7 @@ ModuleAddressSanitizer::getRedzoneSizeForGlobal(uint64_t SizeInBytes) const {
   return RZ;
 }
 
-int ModuleAddressSanitizer::GetAsanVersion(const Module &M) const {
+int ModuleAddressSanitizer::GetAsanVersion() const {
   int LongSize = M.getDataLayout().getPointerSizeInBits();
   bool isAndroid = Triple(M.getTargetTriple()).isAndroid();
   int Version = 8;
@@ -2729,8 +2722,19 @@ int ModuleAddressSanitizer::GetAsanVersion(const Module &M) const {
   return Version;
 }
 
-bool ModuleAddressSanitizer::instrumentModule(Module &M) {
-  initializeCallbacks(M);
+GlobalVariable *ModuleAddressSanitizer::getOrCreateModuleName() {
+  if (!ModuleName) {
+    // We shouldn't merge same module names, as this string serves as unique
+    // module ID in runtime.
+    ModuleName =
+        createPrivateGlobalForString(M, M.getModuleIdentifier(),
+                                     /*AllowMerging*/ false, genName("module"));
+  }
+  return ModuleName;
+}
+
+bool ModuleAddressSanitizer::instrumentModule() {
+  initializeCallbacks();
 
   // Create a module constructor. A destructor is created lazily because not all
   // platforms, and not all modules need it.
@@ -2740,13 +2744,13 @@ bool ModuleAddressSanitizer::instrumentModule(Module &M) {
       // need the init and version check calls.
       AsanCtorFunction = createSanitizerCtor(M, kAsanModuleCtorName);
     } else {
-      std::string AsanVersion = std::to_string(GetAsanVersion(M));
+      std::string AsanVersion = std::to_string(GetAsanVersion());
       std::string VersionCheckName =
           InsertVersionCheck ? (kAsanVersionCheckNamePrefix + AsanVersion) : "";
       std::tie(AsanCtorFunction, std::ignore) =
-          createSanitizerCtorAndInitFunctions(M, kAsanModuleCtorName,
-                                              kAsanInitName, /*InitArgTypes=*/{},
-                                              /*InitArgs=*/{}, VersionCheckName);
+          createSanitizerCtorAndInitFunctions(
+              M, kAsanModuleCtorName, kAsanInitName, /*InitArgTypes=*/{},
+              /*InitArgs=*/{}, VersionCheckName);
     }
   }
 
@@ -2755,10 +2759,10 @@ bool ModuleAddressSanitizer::instrumentModule(Module &M) {
     assert(AsanCtorFunction || ConstructorKind == AsanCtorKind::None);
     if (AsanCtorFunction) {
       IRBuilder<> IRB(AsanCtorFunction->getEntryBlock().getTerminator());
-      instrumentGlobals(IRB, M, &CtorComdat);
+      instrumentGlobals(IRB, &CtorComdat);
     } else {
       IRBuilder<> IRB(*C);
-      instrumentGlobals(IRB, M, &CtorComdat);
+      instrumentGlobals(IRB, &CtorComdat);
     }
   }
 
@@ -2786,7 +2790,7 @@ bool ModuleAddressSanitizer::instrumentModule(Module &M) {
   return true;
 }
 
-void AddressSanitizer::initializeCallbacks(Module &M, const TargetLibraryInfo *TLI) {
+void AddressSanitizer::initializeCallbacks(const TargetLibraryInfo *TLI) {
   IRBuilder<> IRB(*C);
   // Create __asan_report* callbacks.
   // IsWrite, TypeSize and Exp are encoded in the function name.
@@ -2970,7 +2974,7 @@ bool AddressSanitizer::instrumentFunction(Function &F,
 
   LLVM_DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
 
-  initializeCallbacks(*F.getParent(), TLI);
+  initializeCallbacks(TLI);
 
   FunctionStateRAII CleanupObj(this);
 
@@ -3582,7 +3586,7 @@ void FunctionStackPoisoner::processStaticAllocas() {
       IntptrPtrTy);
   GlobalVariable *StackDescriptionGlobal =
       createPrivateGlobalForString(*F.getParent(), DescriptionString,
-                                   /*AllowMerging*/ true, kAsanGenPrefix);
+                                   /*AllowMerging*/ true, genName("stack"));
   Value *Description = IRB.CreatePointerCast(StackDescriptionGlobal, IntptrTy);
   IRB.CreateStore(Description, BasePlus1);
   // Write the PC to redzone[2].
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 1805ea89272ec7f..1b3954a36699a05 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -132,6 +132,11 @@ cl::opt<bool> AtomicFirstCounter(
              "the entry counter)"),
     cl::init(false));
 
+cl::opt<bool> ConditionalCounterUpdate(
+    "conditional-counter-update",
+    cl::desc("Do conditional counter updates in single byte counters mode)"),
+    cl::init(false));
+
 // If the option is not specified, the default behavior about whether
 // counter promotion is done depends on how instrumentaiton lowering
 // pipeline is setup, i.e., the default value of true of this option
@@ -1213,6 +1218,17 @@ Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
 void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
   auto *Addr = getCounterAddress(CoverInstruction);
   IRBuilder<> Builder(CoverInstruction);
+  if (ConditionalCounterUpdate) {
+    Instruction *SplitBefore = CoverInstruction->getNextNode();
+    auto &Ctx = CoverInstruction->getParent()->getContext();
+    auto *Int8Ty = llvm::Type::getInt8Ty(Ctx);
+    Value *Load = Builder.CreateLoad(Int8Ty, Addr, "pgocount");
+    Value *Cmp = Builder.CreateIsNotNull(Load, "pgocount.ifnonzero");
+    Instruction *ThenBranch =
+        SplitBlockAndInsertIfThen(Cmp, SplitBefore, false);
+    Builder.SetInsertPoint(ThenBranch);
+  }
+
   // We store zero to represent that this block is covered.
   Builder.CreateStore(Builder.getInt8(0), Addr);
   CoverInstruction->eraseFromParent();
diff --git a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
index 19f4d8064fc27d2..305bae0e2010d10 100644
--- a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
@@ -91,7 +91,7 @@ BasicBlock::iterator llvm::PrepareToSplitEntryBlock(BasicBlock &BB,
 // Create a constant for Str so that we can pass it to the run-time lib.
 GlobalVariable *llvm::createPrivateGlobalForString(Module &M, StringRef Str,
                                                    bool AllowMerging,
-                                                   const char *NamePrefix) {
+                                                   Twine NamePrefix) {
   Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
   // We use private linkage for module-local strings. If they can be merged
   // with another one, we set the unnamed_addr attribute.
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index 445bf0bb26cc497..4a43120c9a9e7fa 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemoryProfileInfo.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/DataLayout.h"
@@ -753,8 +754,8 @@ stackFrameIncludesInlinedCallStack(ArrayRef<Frame> ProfileCallStack,
   return InlCallStackIter == InlinedCallStack.end();
 }
 
-static bool isNewWithHotColdVariant(Function *Callee,
-                                    const TargetLibraryInfo &TLI) {
+static bool isAllocationWithHotColdVariant(Function *Callee,
+                                           const TargetLibraryInfo &TLI) {
   if (!Callee)
     return false;
   LibFunc Func;
@@ -769,6 +770,8 @@ static bool isNewWithHotColdVariant(Function *Callee,
   case LibFunc_ZnamRKSt9nothrow_t:
   case LibFunc_ZnamSt11align_val_t:
   case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
+  case LibFunc_size_returning_new:
+  case LibFunc_size_returning_new_aligned:
     return true;
   case LibFunc_Znwm12__hot_cold_t:
   case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
@@ -778,6 +781,8 @@ static bool isNewWithHotColdVariant(Function *Callee,
   case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
   case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
   case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
+  case LibFunc_size_returning_new_hot_cold:
+  case LibFunc_size_returning_new_aligned_hot_cold:
     return ClMemProfMatchHotColdNew;
   default:
     return false;
@@ -945,9 +950,8 @@ readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader,
       // instruction's locations match the prefix Frame locations on an
       // allocation context with the same leaf.
       if (AllocInfoIter != LocHashToAllocInfo.end()) {
-        // Only consider allocations via new, to reduce unnecessary metadata,
-        // since those are the only allocations that will be targeted initially.
-        if (!isNewWithHotColdVariant(CI->getCalledFunction(), TLI))
+        // Only consider allocations which support hinting.
+        if (!isAllocationWithHotColdVariant(CI->getCalledFunction(), TLI))
           continue;
         // We may match this instruction's location list to multiple MIB
         // contexts. Add them to a Trie specialized for trimming the contexts to
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 41618194d12ed7c..b3644031c5a44bc 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -110,7 +110,6 @@
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Instrumentation/BlockCoverageInference.h"
 #include "llvm/Transforms/Instrumentation/CFGMST.h"
-#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/MisExpect.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -321,7 +320,6 @@ static cl::opt<unsigned> PGOFunctionCriticalEdgeThreshold(
              " greater than this threshold."));
 
 extern cl::opt<unsigned> MaxNumVTableAnnotations;
-extern cl::opt<std::string> UseCtxProfile;
 
 namespace llvm {
 // Command line option to turn on CFG dot dump after profile annotation.
@@ -339,21 +337,43 @@ extern cl::opt<bool> EnableVTableProfileUse;
 extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
 } // namespace llvm
 
-bool shouldInstrumentForCtxProf() {
-  return PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled() ||
-         !UseCtxProfile.empty();
-}
-bool shouldInstrumentEntryBB() {
-  return PGOInstrumentEntry || shouldInstrumentForCtxProf();
-}
+namespace {
+class FunctionInstrumenter final {
+  Module &M;
+  Function &F;
+  TargetLibraryInfo &TLI;
+  std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
+  BranchProbabilityInfo *const BPI;
+  BlockFrequencyInfo *const BFI;
 
-// FIXME(mtrofin): re-enable this for ctx profiling, for non-indirect calls. Ctx
-// profiling implicitly captures indirect call cases, but not other values.
-// Supporting other values is relatively straight-forward - just another counter
-// range within the context.
-bool isValueProfilingDisabled() {
-  return DisableValueProfiling || shouldInstrumentForCtxProf();
-}
+  const PGOInstrumentationType InstrumentationType;
+
+  // FIXME(mtrofin): re-enable this for ctx profiling, for non-indirect calls.
+  // Ctx profiling implicitly captures indirect call cases, but not other
+  // values. Supporting other values is relatively straight-forward - just
+  // another counter range within the context.
+  bool isValueProfilingDisabled() const {
+    return DisableValueProfiling ||
+           InstrumentationType == PGOInstrumentationType::CTXPROF;
+  }
+
+  bool shouldInstrumentEntryBB() const {
+    return PGOInstrumentEntry ||
+           InstrumentationType == PGOInstrumentationType::CTXPROF;
+  }
+
+public:
+  FunctionInstrumenter(
+      Module &M, Function &F, TargetLibraryInfo &TLI,
+      std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
+      BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr,
+      PGOInstrumentationType InstrumentationType = PGOInstrumentationType::FDO)
+      : M(M), F(F), TLI(TLI), ComdatMembers(ComdatMembers), BPI(BPI), BFI(BFI),
+        InstrumentationType(InstrumentationType) {}
+
+  void instrument();
+};
+} // namespace
 
 // Return a string describing the branch condition that can be
 // used in static branch probability heuristics:
@@ -395,13 +415,16 @@ static const char *ValueProfKindDescr[] = {
 
 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
 // aware this is an ir_level profile so it can set the version flag.
-static GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS) {
+static GlobalVariable *
+createIRLevelProfileFlagVar(Module &M,
+                            PGOInstrumentationType InstrumentationType) {
   const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
   Type *IntTy64 = Type::getInt64Ty(M.getContext());
   uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
-  if (IsCS)
+  if (InstrumentationType == PGOInstrumentationType::CSFDO)
     ProfileVersion |= VARIANT_MASK_CSIR_PROF;
-  if (shouldInstrumentEntryBB())
+  if (PGOInstrumentEntry ||
+      InstrumentationType == PGOInstrumentationType::CTXPROF)
     ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
   if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
     ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
@@ -871,11 +894,7 @@ populateEHOperandBundle(VPCandidateInfo &Cand,
 
 // Visit all edge and instrument the edges not in MST, and do value profiling.
 // Critical edges will be split.
-static void instrumentOneFunc(
-    Function &F, Module *M, TargetLibraryInfo &TLI, BranchProbabilityInfo *BPI,
-    BlockFrequencyInfo *BFI,
-    std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
-    bool IsCS) {
+void FunctionInstrumenter::instrument() {
   if (!PGOBlockCoverage) {
     // Split indirectbr critical edges here before computing the MST rather than
     // later in getInstrBB() to avoid invalidating it.
@@ -883,19 +902,20 @@ static void instrumentOneFunc(
   }
 
   FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
-      F, TLI, ComdatMembers, true, BPI, BFI, IsCS, shouldInstrumentEntryBB(),
-      PGOBlockCoverage);
+      F, TLI, ComdatMembers, true, BPI, BFI,
+      InstrumentationType == PGOInstrumentationType::CSFDO,
+      shouldInstrumentEntryBB(), PGOBlockCoverage);
 
   auto Name = FuncInfo.FuncNameVar;
-  auto CFGHash = ConstantInt::get(Type::getInt64Ty(M->getContext()),
-                                  FuncInfo.FunctionHash);
+  auto CFGHash =
+      ConstantInt::get(Type::getInt64Ty(M.getContext()), FuncInfo.FunctionHash);
   if (PGOFunctionEntryCoverage) {
     auto &EntryBB = F.getEntryBlock();
     IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());
     // llvm.instrprof.cover(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                      i32 <index>)
     Builder.CreateCall(
-        Intrinsic::getDeclaration(M, Intrinsic::instrprof_cover),
+        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_cover),
         {Name, CFGHash, Builder.getInt32(1), Builder.getInt32(0)});
     return;
   }
@@ -905,9 +925,9 @@ static void instrumentOneFunc(
   unsigned NumCounters =
       InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
 
-  if (shouldInstrumentForCtxProf()) {
+  if (InstrumentationType == PGOInstrumentationType::CTXPROF) {
     auto *CSIntrinsic =
-        Intrinsic::getDeclaration(M, Intrinsic::instrprof_callsite);
+        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_callsite);
     // We want to count the instrumentable callsites, then instrument them. This
     // is because the llvm.instrprof.callsite intrinsic has an argument (like
     // the other instrprof intrinsics) capturing the total number of
@@ -950,7 +970,7 @@ static void instrumentOneFunc(
     // llvm.instrprof.timestamp(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                          i32 <index>)
     Builder.CreateCall(
-        Intrinsic::getDeclaration(M, Intrinsic::instrprof_timestamp),
+        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_timestamp),
         {Name, CFGHash, Builder.getInt32(NumCounters), Builder.getInt32(I)});
     I += PGOBlockCoverage ? 8 : 1;
   }
@@ -962,9 +982,9 @@ static void instrumentOneFunc(
     // llvm.instrprof.increment(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                          i32 <index>)
     Builder.CreateCall(
-        Intrinsic::getDeclaration(M, PGOBlockCoverage
-                                         ? Intrinsic::instrprof_cover
-                                         : Intrinsic::instrprof_increment),
+        Intrinsic::getDeclaration(&M, PGOBlockCoverage
+                                          ? Intrinsic::instrprof_cover
+                                          : Intrinsic::instrprof_increment),
         {Name, CFGHash, Builder.getInt32(NumCounters), Builder.getInt32(I++)});
   }
 
@@ -1011,7 +1031,7 @@ static void instrumentOneFunc(
       SmallVector<OperandBundleDef, 1> OpBundles;
       populateEHOperandBundle(Cand, BlockColors, OpBundles);
       Builder.CreateCall(
-          Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
+          Intrinsic::getDeclaration(&M, Intrinsic::instrprof_value_profile),
           {FuncInfo.FuncNameVar, Builder.getInt64(FuncInfo.FunctionHash),
            ToProfile, Builder.getInt32(Kind), Builder.getInt32(SiteIndex++)},
           OpBundles);
@@ -1746,7 +1766,7 @@ static uint32_t getMaxNumAnnotations(InstrProfValueKind ValueProfKind) {
 
 // Traverse all valuesites and annotate the instructions for all value kind.
 void PGOUseFunc::annotateValueSites() {
-  if (isValueProfilingDisabled())
+  if (DisableValueProfiling)
     return;
 
   // Create the PGOFuncName meta data.
@@ -1861,11 +1881,12 @@ static bool skipPGOGen(const Function &F) {
 static bool InstrumentAllFunctions(
     Module &M, function_ref<TargetLibraryInfo &(Function &)> LookupTLI,
     function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
-    function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) {
+    function_ref<BlockFrequencyInfo *(Function &)> LookupBFI,
+    PGOInstrumentationType InstrumentationType) {
   // For the context-sensitve instrumentation, we should have a separated pass
   // (before LTO/ThinLTO linking) to create these variables.
-  if (!IsCS && !shouldInstrumentForCtxProf())
-    createIRLevelProfileFlagVar(M, /*IsCS=*/false);
+  if (InstrumentationType == PGOInstrumentationType::FDO)
+    createIRLevelProfileFlagVar(M, InstrumentationType);
 
   Triple TT(M.getTargetTriple());
   LLVMContext &Ctx = M.getContext();
@@ -1884,7 +1905,9 @@ static bool InstrumentAllFunctions(
     auto &TLI = LookupTLI(F);
     auto *BPI = LookupBPI(F);
     auto *BFI = LookupBFI(F);
-    instrumentOneFunc(F, &M, TLI, BPI, BFI, ComdatMembers, IsCS);
+    FunctionInstrumenter FI(M, F, TLI, ComdatMembers, BPI, BFI,
+                            InstrumentationType);
+    FI.instrument();
   }
   return true;
 }
@@ -1894,7 +1917,8 @@ PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &MAM) {
   createProfileFileNameVar(M, CSInstrName);
   // The variable in a comdat may be discarded by LTO. Ensure the declaration
   // will be retained.
-  appendToCompilerUsed(M, createIRLevelProfileFlagVar(M, /*IsCS=*/true));
+  appendToCompilerUsed(
+      M, createIRLevelProfileFlagVar(M, PGOInstrumentationType::CSFDO));
   if (ProfileSampling)
     createProfileSamplingVar(M);
   PreservedAnalyses PA;
@@ -1916,7 +1940,8 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
     return &FAM.getResult<BlockFrequencyAnalysis>(F);
   };
 
-  if (!InstrumentAllFunctions(M, LookupTLI, LookupBPI, LookupBFI, IsCS))
+  if (!InstrumentAllFunctions(M, LookupTLI, LookupBPI, LookupBFI,
+                              InstrumentationType))
     return PreservedAnalyses::all();
 
   return PreservedAnalyses::none();
@@ -2115,7 +2140,6 @@ static bool annotateAllFunctions(
   bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
   if (PGOInstrumentEntry.getNumOccurrences() > 0)
     InstrumentFuncEntry = PGOInstrumentEntry;
-  InstrumentFuncEntry |= shouldInstrumentForCtxProf();
 
   bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
   for (auto &F : M) {
diff --git a/llvm/lib/Transforms/Scalar/CMakeLists.txt b/llvm/lib/Transforms/Scalar/CMakeLists.txt
index ba09ebf8b04c4cd..939a1457239567a 100644
--- a/llvm/lib/Transforms/Scalar/CMakeLists.txt
+++ b/llvm/lib/Transforms/Scalar/CMakeLists.txt
@@ -44,6 +44,7 @@ add_llvm_component_library(LLVMScalarOpts
   LoopRotation.cpp
   LoopSimplifyCFG.cpp
   LoopStrengthReduce.cpp
+  LoopTermFold.cpp
   LoopUnrollPass.cpp
   LoopUnrollAndJamPass.cpp
   LoopVersioningLICM.cpp
diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
index a5571409bba6824..3e3e5bfe2d63327 100644
--- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -240,7 +240,8 @@ class InferAddressSpacesImpl {
       SmallVectorImpl<const Use *> *PoisonUsesToFix) const;
   unsigned joinAddressSpaces(unsigned AS1, unsigned AS2) const;
 
-  unsigned getPredicatedAddrSpace(const Value &V, Value *Opnd) const;
+  unsigned getPredicatedAddrSpace(const Value &PtrV,
+                                  const Value *UserCtx) const;
 
 public:
   InferAddressSpacesImpl(AssumptionCache &AC, const DominatorTree *DT,
@@ -909,18 +910,19 @@ void InferAddressSpacesImpl::inferAddressSpaces(
   }
 }
 
-unsigned InferAddressSpacesImpl::getPredicatedAddrSpace(const Value &V,
-                                                        Value *Opnd) const {
-  const Instruction *I = dyn_cast<Instruction>(&V);
-  if (!I)
+unsigned
+InferAddressSpacesImpl::getPredicatedAddrSpace(const Value &Ptr,
+                                               const Value *UserCtx) const {
+  const Instruction *UserCtxI = dyn_cast<Instruction>(UserCtx);
+  if (!UserCtxI)
     return UninitializedAddressSpace;
 
-  Opnd = Opnd->stripInBoundsOffsets();
-  for (auto &AssumeVH : AC.assumptionsFor(Opnd)) {
+  const Value *StrippedPtr = Ptr.stripInBoundsOffsets();
+  for (auto &AssumeVH : AC.assumptionsFor(StrippedPtr)) {
     if (!AssumeVH)
       continue;
     CallInst *CI = cast<CallInst>(AssumeVH);
-    if (!isValidAssumeForContext(CI, I, DT))
+    if (!isValidAssumeForContext(CI, UserCtxI, DT))
       continue;
 
     const Value *Ptr;
@@ -989,7 +991,7 @@ bool InferAddressSpacesImpl::updateAddressSpace(
           OperandAS = PtrOperand->getType()->getPointerAddressSpace();
           if (OperandAS == FlatAddrSpace) {
             // Check AC for assumption dominating V.
-            unsigned AS = getPredicatedAddrSpace(V, PtrOperand);
+            unsigned AS = getPredicatedAddrSpace(*PtrOperand, &V);
             if (AS != UninitializedAddressSpace) {
               LLVM_DEBUG(dbgs()
                          << "  deduce operand AS from the predicate addrspace "
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 91461d1ed27592c..a62b87fe2a53d46 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -189,10 +189,6 @@ static cl::opt<unsigned> SetupCostDepthLimit(
     "lsr-setupcost-depth-limit", cl::Hidden, cl::init(7),
     cl::desc("The limit on recursion depth for LSRs setup cost"));
 
-static cl::opt<cl::boolOrDefault> AllowTerminatingConditionFoldingAfterLSR(
-    "lsr-term-fold", cl::Hidden,
-    cl::desc("Attempt to replace primary IV with other IV."));
-
 static cl::opt<cl::boolOrDefault> AllowDropSolutionIfLessProfitable(
     "lsr-drop-solution", cl::Hidden,
     cl::desc("Attempt to drop solution if it is less profitable"));
@@ -205,9 +201,6 @@ static cl::opt<bool> DropScaledForVScale(
     "lsr-drop-scaled-reg-for-vscale", cl::Hidden, cl::init(true),
     cl::desc("Avoid using scaled registers with vscale-relative addressing"));
 
-STATISTIC(NumTermFold,
-          "Number of terminating condition fold recognized and performed");
-
 #ifndef NDEBUG
 // Stress test IV chain generation.
 static cl::opt<bool> StressIVChain(
@@ -7062,186 +7055,6 @@ static llvm::PHINode *GetInductionVariable(const Loop &L, ScalarEvolution &SE,
   return nullptr;
 }
 
-static std::optional<std::tuple<PHINode *, PHINode *, const SCEV *, bool>>
-canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
-                      const LoopInfo &LI, const TargetTransformInfo &TTI) {
-  if (!L->isInnermost()) {
-    LLVM_DEBUG(dbgs() << "Cannot fold on non-innermost loop\n");
-    return std::nullopt;
-  }
-  // Only inspect on simple loop structure
-  if (!L->isLoopSimplifyForm()) {
-    LLVM_DEBUG(dbgs() << "Cannot fold on non-simple loop\n");
-    return std::nullopt;
-  }
-
-  if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
-    LLVM_DEBUG(dbgs() << "Cannot fold on backedge that is loop variant\n");
-    return std::nullopt;
-  }
-
-  BasicBlock *LoopLatch = L->getLoopLatch();
-  BranchInst *BI = dyn_cast<BranchInst>(LoopLatch->getTerminator());
-  if (!BI || BI->isUnconditional())
-    return std::nullopt;
-  auto *TermCond = dyn_cast<ICmpInst>(BI->getCondition());
-  if (!TermCond) {
-    LLVM_DEBUG(
-        dbgs() << "Cannot fold on branching condition that is not an ICmpInst");
-    return std::nullopt;
-  }
-  if (!TermCond->hasOneUse()) {
-    LLVM_DEBUG(
-        dbgs()
-        << "Cannot replace terminating condition with more than one use\n");
-    return std::nullopt;
-  }
-
-  BinaryOperator *LHS = dyn_cast<BinaryOperator>(TermCond->getOperand(0));
-  Value *RHS = TermCond->getOperand(1);
-  if (!LHS || !L->isLoopInvariant(RHS))
-    // We could pattern match the inverse form of the icmp, but that is
-    // non-canonical, and this pass is running *very* late in the pipeline.
-    return std::nullopt;
-
-  // Find the IV used by the current exit condition.
-  PHINode *ToFold;
-  Value *ToFoldStart, *ToFoldStep;
-  if (!matchSimpleRecurrence(LHS, ToFold, ToFoldStart, ToFoldStep))
-    return std::nullopt;
-
-  // Ensure the simple recurrence is a part of the current loop.
-  if (ToFold->getParent() != L->getHeader())
-    return std::nullopt;
-
-  // If that IV isn't dead after we rewrite the exit condition in terms of
-  // another IV, there's no point in doing the transform.
-  if (!isAlmostDeadIV(ToFold, LoopLatch, TermCond))
-    return std::nullopt;
-
-  // Inserting instructions in the preheader has a runtime cost, scale
-  // the allowed cost with the loops trip count as best we can.
-  const unsigned ExpansionBudget = [&]() {
-    unsigned Budget = 2 * SCEVCheapExpansionBudget;
-    if (unsigned SmallTC = SE.getSmallConstantMaxTripCount(L))
-      return std::min(Budget, SmallTC);
-    if (std::optional<unsigned> SmallTC = getLoopEstimatedTripCount(L))
-      return std::min(Budget, *SmallTC);
-    // Unknown trip count, assume long running by default.
-    return Budget;
-  }();
-
-  const SCEV *BECount = SE.getBackedgeTakenCount(L);
-  const DataLayout &DL = L->getHeader()->getDataLayout();
-  SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
-
-  PHINode *ToHelpFold = nullptr;
-  const SCEV *TermValueS = nullptr;
-  bool MustDropPoison = false;
-  auto InsertPt = L->getLoopPreheader()->getTerminator();
-  for (PHINode &PN : L->getHeader()->phis()) {
-    if (ToFold == &PN)
-      continue;
-
-    if (!SE.isSCEVable(PN.getType())) {
-      LLVM_DEBUG(dbgs() << "IV of phi '" << PN
-                        << "' is not SCEV-able, not qualified for the "
-                           "terminating condition folding.\n");
-      continue;
-    }
-    const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(&PN));
-    // Only speculate on affine AddRec
-    if (!AddRec || !AddRec->isAffine()) {
-      LLVM_DEBUG(dbgs() << "SCEV of phi '" << PN
-                        << "' is not an affine add recursion, not qualified "
-                           "for the terminating condition folding.\n");
-      continue;
-    }
-
-    // Check that we can compute the value of AddRec on the exiting iteration
-    // without soundness problems.  evaluateAtIteration internally needs
-    // to multiply the stride of the iteration number - which may wrap around.
-    // The issue here is subtle because computing the result accounting for
-    // wrap is insufficient. In order to use the result in an exit test, we
-    // must also know that AddRec doesn't take the same value on any previous
-    // iteration. The simplest case to consider is a candidate IV which is
-    // narrower than the trip count (and thus original IV), but this can
-    // also happen due to non-unit strides on the candidate IVs.
-    if (!AddRec->hasNoSelfWrap() ||
-        !SE.isKnownNonZero(AddRec->getStepRecurrence(SE)))
-      continue;
-
-    const SCEVAddRecExpr *PostInc = AddRec->getPostIncExpr(SE);
-    const SCEV *TermValueSLocal = PostInc->evaluateAtIteration(BECount, SE);
-    if (!Expander.isSafeToExpand(TermValueSLocal)) {
-      LLVM_DEBUG(
-          dbgs() << "Is not safe to expand terminating value for phi node" << PN
-                 << "\n");
-      continue;
-    }
-
-    if (Expander.isHighCostExpansion(TermValueSLocal, L, ExpansionBudget,
-                                     &TTI, InsertPt)) {
-      LLVM_DEBUG(
-          dbgs() << "Is too expensive to expand terminating value for phi node"
-                 << PN << "\n");
-      continue;
-    }
-
-    // The candidate IV may have been otherwise dead and poison from the
-    // very first iteration.  If we can't disprove that, we can't use the IV.
-    if (!mustExecuteUBIfPoisonOnPathTo(&PN, LoopLatch->getTerminator(), &DT)) {
-      LLVM_DEBUG(dbgs() << "Can not prove poison safety for IV "
-                        << PN << "\n");
-      continue;
-    }
-
-    // The candidate IV may become poison on the last iteration.  If this
-    // value is not branched on, this is a well defined program.  We're
-    // about to add a new use to this IV, and we have to ensure we don't
-    // insert UB which didn't previously exist.
-    bool MustDropPoisonLocal = false;
-    Instruction *PostIncV =
-      cast<Instruction>(PN.getIncomingValueForBlock(LoopLatch));
-    if (!mustExecuteUBIfPoisonOnPathTo(PostIncV, LoopLatch->getTerminator(),
-                                       &DT)) {
-      LLVM_DEBUG(dbgs() << "Can not prove poison safety to insert use"
-                        << PN << "\n");
-
-      // If this is a complex recurrance with multiple instructions computing
-      // the backedge value, we might need to strip poison flags from all of
-      // them.
-      if (PostIncV->getOperand(0) != &PN)
-        continue;
-
-      // In order to perform the transform, we need to drop the poison generating
-      // flags on this instruction (if any).
-      MustDropPoisonLocal = PostIncV->hasPoisonGeneratingFlags();
-    }
-
-    // We pick the last legal alternate IV.  We could expore choosing an optimal
-    // alternate IV if we had a decent heuristic to do so.
-    ToHelpFold = &PN;
-    TermValueS = TermValueSLocal;
-    MustDropPoison = MustDropPoisonLocal;
-  }
-
-  LLVM_DEBUG(if (ToFold && !ToHelpFold) dbgs()
-                 << "Cannot find other AddRec IV to help folding\n";);
-
-  LLVM_DEBUG(if (ToFold && ToHelpFold) dbgs()
-             << "\nFound loop that can fold terminating condition\n"
-             << "  BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n"
-             << "  TermCond: " << *TermCond << "\n"
-             << "  BrandInst: " << *BI << "\n"
-             << "  ToFold: " << *ToFold << "\n"
-             << "  ToHelpFold: " << *ToHelpFold << "\n");
-
-  if (!ToFold || !ToHelpFold)
-    return std::nullopt;
-  return std::make_tuple(ToFold, ToHelpFold, TermValueS, MustDropPoison);
-}
-
 static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
                                DominatorTree &DT, LoopInfo &LI,
                                const TargetTransformInfo &TTI,
@@ -7302,81 +7115,6 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
     }
   }
 
-  const bool EnableFormTerm = [&] {
-    switch (AllowTerminatingConditionFoldingAfterLSR) {
-    case cl::BOU_TRUE:
-      return true;
-    case cl::BOU_FALSE:
-      return false;
-    case cl::BOU_UNSET:
-      return TTI.shouldFoldTerminatingConditionAfterLSR();
-    }
-    llvm_unreachable("Unhandled cl::boolOrDefault enum");
-  }();
-
-  if (EnableFormTerm) {
-    if (auto Opt = canFoldTermCondOfLoop(L, SE, DT, LI, TTI)) {
-      auto [ToFold, ToHelpFold, TermValueS, MustDrop] = *Opt;
-
-      Changed = true;
-      NumTermFold++;
-
-      BasicBlock *LoopPreheader = L->getLoopPreheader();
-      BasicBlock *LoopLatch = L->getLoopLatch();
-
-      (void)ToFold;
-      LLVM_DEBUG(dbgs() << "To fold phi-node:\n"
-                        << *ToFold << "\n"
-                        << "New term-cond phi-node:\n"
-                        << *ToHelpFold << "\n");
-
-      Value *StartValue = ToHelpFold->getIncomingValueForBlock(LoopPreheader);
-      (void)StartValue;
-      Value *LoopValue = ToHelpFold->getIncomingValueForBlock(LoopLatch);
-
-      // See comment in canFoldTermCondOfLoop on why this is sufficient.
-      if (MustDrop)
-        cast<Instruction>(LoopValue)->dropPoisonGeneratingFlags();
-
-      // SCEVExpander for both use in preheader and latch
-      const DataLayout &DL = L->getHeader()->getDataLayout();
-      SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
-
-      assert(Expander.isSafeToExpand(TermValueS) &&
-             "Terminating value was checked safe in canFoldTerminatingCondition");
-
-      // Create new terminating value at loop preheader
-      Value *TermValue = Expander.expandCodeFor(TermValueS, ToHelpFold->getType(),
-                                                LoopPreheader->getTerminator());
-
-      LLVM_DEBUG(dbgs() << "Start value of new term-cond phi-node:\n"
-                        << *StartValue << "\n"
-                        << "Terminating value of new term-cond phi-node:\n"
-                        << *TermValue << "\n");
-
-      // Create new terminating condition at loop latch
-      BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator());
-      ICmpInst *OldTermCond = cast<ICmpInst>(BI->getCondition());
-      IRBuilder<> LatchBuilder(LoopLatch->getTerminator());
-      Value *NewTermCond =
-          LatchBuilder.CreateICmp(CmpInst::ICMP_EQ, LoopValue, TermValue,
-                                  "lsr_fold_term_cond.replaced_term_cond");
-      // Swap successors to exit loop body if IV equals to new TermValue
-      if (BI->getSuccessor(0) == L->getHeader())
-        BI->swapSuccessors();
-
-      LLVM_DEBUG(dbgs() << "Old term-cond:\n"
-                        << *OldTermCond << "\n"
-                        << "New term-cond:\n" << *NewTermCond << "\n");
-
-      BI->setCondition(NewTermCond);
-
-      Expander.clear();
-      OldTermCond->eraseFromParent();
-      DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
-    }
-  }
-
   if (SalvageableDVIRecords.empty())
     return Changed;
 
diff --git a/llvm/lib/Transforms/Scalar/LoopTermFold.cpp b/llvm/lib/Transforms/Scalar/LoopTermFold.cpp
new file mode 100644
index 000000000000000..12ef367adc43e36
--- /dev/null
+++ b/llvm/lib/Transforms/Scalar/LoopTermFold.cpp
@@ -0,0 +1,379 @@
+//===- LoopTermFold.cpp - Eliminate last use of IV in exit branch----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Scalar/LoopTermFold.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+#include <cassert>
+#include <optional>
+#include <utility>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "loop-term-fold"
+
+STATISTIC(NumTermFold,
+          "Number of terminating condition fold recognized and performed");
+
+static std::optional<std::tuple<PHINode *, PHINode *, const SCEV *, bool>>
+canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
+                      const LoopInfo &LI, const TargetTransformInfo &TTI) {
+  if (!L->isInnermost()) {
+    LLVM_DEBUG(dbgs() << "Cannot fold on non-innermost loop\n");
+    return std::nullopt;
+  }
+  // Only inspect on simple loop structure
+  if (!L->isLoopSimplifyForm()) {
+    LLVM_DEBUG(dbgs() << "Cannot fold on non-simple loop\n");
+    return std::nullopt;
+  }
+
+  if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
+    LLVM_DEBUG(dbgs() << "Cannot fold on backedge that is loop variant\n");
+    return std::nullopt;
+  }
+
+  BasicBlock *LoopLatch = L->getLoopLatch();
+  BranchInst *BI = dyn_cast<BranchInst>(LoopLatch->getTerminator());
+  if (!BI || BI->isUnconditional())
+    return std::nullopt;
+  auto *TermCond = dyn_cast<ICmpInst>(BI->getCondition());
+  if (!TermCond) {
+    LLVM_DEBUG(
+        dbgs() << "Cannot fold on branching condition that is not an ICmpInst");
+    return std::nullopt;
+  }
+  if (!TermCond->hasOneUse()) {
+    LLVM_DEBUG(
+        dbgs()
+        << "Cannot replace terminating condition with more than one use\n");
+    return std::nullopt;
+  }
+
+  BinaryOperator *LHS = dyn_cast<BinaryOperator>(TermCond->getOperand(0));
+  Value *RHS = TermCond->getOperand(1);
+  if (!LHS || !L->isLoopInvariant(RHS))
+    // We could pattern match the inverse form of the icmp, but that is
+    // non-canonical, and this pass is running *very* late in the pipeline.
+    return std::nullopt;
+
+  // Find the IV used by the current exit condition.
+  PHINode *ToFold;
+  Value *ToFoldStart, *ToFoldStep;
+  if (!matchSimpleRecurrence(LHS, ToFold, ToFoldStart, ToFoldStep))
+    return std::nullopt;
+
+  // Ensure the simple recurrence is a part of the current loop.
+  if (ToFold->getParent() != L->getHeader())
+    return std::nullopt;
+
+  // If that IV isn't dead after we rewrite the exit condition in terms of
+  // another IV, there's no point in doing the transform.
+  if (!isAlmostDeadIV(ToFold, LoopLatch, TermCond))
+    return std::nullopt;
+
+  // Inserting instructions in the preheader has a runtime cost, scale
+  // the allowed cost with the loops trip count as best we can.
+  const unsigned ExpansionBudget = [&]() {
+    unsigned Budget = 2 * SCEVCheapExpansionBudget;
+    if (unsigned SmallTC = SE.getSmallConstantMaxTripCount(L))
+      return std::min(Budget, SmallTC);
+    if (std::optional<unsigned> SmallTC = getLoopEstimatedTripCount(L))
+      return std::min(Budget, *SmallTC);
+    // Unknown trip count, assume long running by default.
+    return Budget;
+  }();
+
+  const SCEV *BECount = SE.getBackedgeTakenCount(L);
+  const DataLayout &DL = L->getHeader()->getDataLayout();
+  SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
+
+  PHINode *ToHelpFold = nullptr;
+  const SCEV *TermValueS = nullptr;
+  bool MustDropPoison = false;
+  auto InsertPt = L->getLoopPreheader()->getTerminator();
+  for (PHINode &PN : L->getHeader()->phis()) {
+    if (ToFold == &PN)
+      continue;
+
+    if (!SE.isSCEVable(PN.getType())) {
+      LLVM_DEBUG(dbgs() << "IV of phi '" << PN
+                        << "' is not SCEV-able, not qualified for the "
+                           "terminating condition folding.\n");
+      continue;
+    }
+    const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(&PN));
+    // Only speculate on affine AddRec
+    if (!AddRec || !AddRec->isAffine()) {
+      LLVM_DEBUG(dbgs() << "SCEV of phi '" << PN
+                        << "' is not an affine add recursion, not qualified "
+                           "for the terminating condition folding.\n");
+      continue;
+    }
+
+    // Check that we can compute the value of AddRec on the exiting iteration
+    // without soundness problems.  evaluateAtIteration internally needs
+    // to multiply the stride of the iteration number - which may wrap around.
+    // The issue here is subtle because computing the result accounting for
+    // wrap is insufficient. In order to use the result in an exit test, we
+    // must also know that AddRec doesn't take the same value on any previous
+    // iteration. The simplest case to consider is a candidate IV which is
+    // narrower than the trip count (and thus original IV), but this can
+    // also happen due to non-unit strides on the candidate IVs.
+    if (!AddRec->hasNoSelfWrap() ||
+        !SE.isKnownNonZero(AddRec->getStepRecurrence(SE)))
+      continue;
+
+    const SCEVAddRecExpr *PostInc = AddRec->getPostIncExpr(SE);
+    const SCEV *TermValueSLocal = PostInc->evaluateAtIteration(BECount, SE);
+    if (!Expander.isSafeToExpand(TermValueSLocal)) {
+      LLVM_DEBUG(
+          dbgs() << "Is not safe to expand terminating value for phi node" << PN
+                 << "\n");
+      continue;
+    }
+
+    if (Expander.isHighCostExpansion(TermValueSLocal, L, ExpansionBudget, &TTI,
+                                     InsertPt)) {
+      LLVM_DEBUG(
+          dbgs() << "Is too expensive to expand terminating value for phi node"
+                 << PN << "\n");
+      continue;
+    }
+
+    // The candidate IV may have been otherwise dead and poison from the
+    // very first iteration.  If we can't disprove that, we can't use the IV.
+    if (!mustExecuteUBIfPoisonOnPathTo(&PN, LoopLatch->getTerminator(), &DT)) {
+      LLVM_DEBUG(dbgs() << "Can not prove poison safety for IV " << PN << "\n");
+      continue;
+    }
+
+    // The candidate IV may become poison on the last iteration.  If this
+    // value is not branched on, this is a well defined program.  We're
+    // about to add a new use to this IV, and we have to ensure we don't
+    // insert UB which didn't previously exist.
+    bool MustDropPoisonLocal = false;
+    Instruction *PostIncV =
+        cast<Instruction>(PN.getIncomingValueForBlock(LoopLatch));
+    if (!mustExecuteUBIfPoisonOnPathTo(PostIncV, LoopLatch->getTerminator(),
+                                       &DT)) {
+      LLVM_DEBUG(dbgs() << "Can not prove poison safety to insert use" << PN
+                        << "\n");
+
+      // If this is a complex recurrance with multiple instructions computing
+      // the backedge value, we might need to strip poison flags from all of
+      // them.
+      if (PostIncV->getOperand(0) != &PN)
+        continue;
+
+      // In order to perform the transform, we need to drop the poison
+      // generating flags on this instruction (if any).
+      MustDropPoisonLocal = PostIncV->hasPoisonGeneratingFlags();
+    }
+
+    // We pick the last legal alternate IV.  We could expore choosing an optimal
+    // alternate IV if we had a decent heuristic to do so.
+    ToHelpFold = &PN;
+    TermValueS = TermValueSLocal;
+    MustDropPoison = MustDropPoisonLocal;
+  }
+
+  LLVM_DEBUG(if (ToFold && !ToHelpFold) dbgs()
+                 << "Cannot find other AddRec IV to help folding\n";);
+
+  LLVM_DEBUG(if (ToFold && ToHelpFold) dbgs()
+             << "\nFound loop that can fold terminating condition\n"
+             << "  BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n"
+             << "  TermCond: " << *TermCond << "\n"
+             << "  BrandInst: " << *BI << "\n"
+             << "  ToFold: " << *ToFold << "\n"
+             << "  ToHelpFold: " << *ToHelpFold << "\n");
+
+  if (!ToFold || !ToHelpFold)
+    return std::nullopt;
+  return std::make_tuple(ToFold, ToHelpFold, TermValueS, MustDropPoison);
+}
+
+static bool RunTermFold(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
+                        LoopInfo &LI, const TargetTransformInfo &TTI,
+                        TargetLibraryInfo &TLI, MemorySSA *MSSA) {
+  std::unique_ptr<MemorySSAUpdater> MSSAU;
+  if (MSSA)
+    MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
+
+  auto Opt = canFoldTermCondOfLoop(L, SE, DT, LI, TTI);
+  if (!Opt)
+    return false;
+
+  auto [ToFold, ToHelpFold, TermValueS, MustDrop] = *Opt;
+
+  NumTermFold++;
+
+  BasicBlock *LoopPreheader = L->getLoopPreheader();
+  BasicBlock *LoopLatch = L->getLoopLatch();
+
+  (void)ToFold;
+  LLVM_DEBUG(dbgs() << "To fold phi-node:\n"
+                    << *ToFold << "\n"
+                    << "New term-cond phi-node:\n"
+                    << *ToHelpFold << "\n");
+
+  Value *StartValue = ToHelpFold->getIncomingValueForBlock(LoopPreheader);
+  (void)StartValue;
+  Value *LoopValue = ToHelpFold->getIncomingValueForBlock(LoopLatch);
+
+  // See comment in canFoldTermCondOfLoop on why this is sufficient.
+  if (MustDrop)
+    cast<Instruction>(LoopValue)->dropPoisonGeneratingFlags();
+
+  // SCEVExpander for both use in preheader and latch
+  const DataLayout &DL = L->getHeader()->getDataLayout();
+  SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
+
+  assert(Expander.isSafeToExpand(TermValueS) &&
+         "Terminating value was checked safe in canFoldTerminatingCondition");
+
+  // Create new terminating value at loop preheader
+  Value *TermValue = Expander.expandCodeFor(TermValueS, ToHelpFold->getType(),
+                                            LoopPreheader->getTerminator());
+
+  LLVM_DEBUG(dbgs() << "Start value of new term-cond phi-node:\n"
+                    << *StartValue << "\n"
+                    << "Terminating value of new term-cond phi-node:\n"
+                    << *TermValue << "\n");
+
+  // Create new terminating condition at loop latch
+  BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator());
+  ICmpInst *OldTermCond = cast<ICmpInst>(BI->getCondition());
+  IRBuilder<> LatchBuilder(LoopLatch->getTerminator());
+  Value *NewTermCond =
+      LatchBuilder.CreateICmp(CmpInst::ICMP_EQ, LoopValue, TermValue,
+                              "lsr_fold_term_cond.replaced_term_cond");
+  // Swap successors to exit loop body if IV equals to new TermValue
+  if (BI->getSuccessor(0) == L->getHeader())
+    BI->swapSuccessors();
+
+  LLVM_DEBUG(dbgs() << "Old term-cond:\n"
+                    << *OldTermCond << "\n"
+                    << "New term-cond:\n"
+                    << *NewTermCond << "\n");
+
+  BI->setCondition(NewTermCond);
+
+  Expander.clear();
+  OldTermCond->eraseFromParent();
+  DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
+  return true;
+}
+
+namespace {
+
+class LoopTermFold : public LoopPass {
+public:
+  static char ID; // Pass ID, replacement for typeid
+
+  LoopTermFold();
+
+private:
+  bool runOnLoop(Loop *L, LPPassManager &LPM) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+
+} // end anonymous namespace
+
+LoopTermFold::LoopTermFold() : LoopPass(ID) {
+  initializeLoopTermFoldPass(*PassRegistry::getPassRegistry());
+}
+
+void LoopTermFold::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<LoopInfoWrapperPass>();
+  AU.addPreserved<LoopInfoWrapperPass>();
+  AU.addPreservedID(LoopSimplifyID);
+  AU.addRequiredID(LoopSimplifyID);
+  AU.addRequired<DominatorTreeWrapperPass>();
+  AU.addPreserved<DominatorTreeWrapperPass>();
+  AU.addRequired<ScalarEvolutionWrapperPass>();
+  AU.addPreserved<ScalarEvolutionWrapperPass>();
+  AU.addRequired<TargetLibraryInfoWrapperPass>();
+  AU.addRequired<TargetTransformInfoWrapperPass>();
+  AU.addPreserved<MemorySSAWrapperPass>();
+}
+
+bool LoopTermFold::runOnLoop(Loop *L, LPPassManager & /*LPM*/) {
+  if (skipLoop(L))
+    return false;
+
+  auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
+  auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+  const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
+      *L->getHeader()->getParent());
+  auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
+      *L->getHeader()->getParent());
+  auto *MSSAAnalysis = getAnalysisIfAvailable<MemorySSAWrapperPass>();
+  MemorySSA *MSSA = nullptr;
+  if (MSSAAnalysis)
+    MSSA = &MSSAAnalysis->getMSSA();
+  return RunTermFold(L, SE, DT, LI, TTI, TLI, MSSA);
+}
+
+PreservedAnalyses LoopTermFoldPass::run(Loop &L, LoopAnalysisManager &AM,
+                                        LoopStandardAnalysisResults &AR,
+                                        LPMUpdater &) {
+  if (!RunTermFold(&L, AR.SE, AR.DT, AR.LI, AR.TTI, AR.TLI, AR.MSSA))
+    return PreservedAnalyses::all();
+
+  auto PA = getLoopPassPreservedAnalyses();
+  if (AR.MSSA)
+    PA.preserve<MemorySSAAnalysis>();
+  return PA;
+}
+
+char LoopTermFold::ID = 0;
+
+INITIALIZE_PASS_BEGIN(LoopTermFold, "loop-term-fold", "Loop Terminator Folding",
+                      false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
+INITIALIZE_PASS_END(LoopTermFold, "loop-term-fold", "Loop Terminator Folding",
+                    false, false)
+
+Pass *llvm::createLoopTermFoldPass() { return new LoopTermFold(); }
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index 86669e8c5aa49b1..7aeee1d31f7e791 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -30,6 +30,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeLegacyLICMPassPass(Registry);
   initializeLoopDataPrefetchLegacyPassPass(Registry);
   initializeLoopStrengthReducePass(Registry);
+  initializeLoopTermFoldPass(Registry);
   initializeLoopUnrollPass(Registry);
   initializeLowerAtomicLegacyPassPass(Registry);
   initializeMergeICmpsLegacyPassPass(Registry);
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index 8eadf8900020d9e..9cb7bad94c20bcd 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
@@ -161,7 +162,9 @@ static void scalarizeMaskedLoad(const DataLayout &DL, CallInst *CI,
 
   // Short-cut if the mask is all-true.
   if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
-    Value *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
+    LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
+    NewI->copyMetadata(*CI);
+    NewI->takeName(CI);
     CI->replaceAllUsesWith(NewI);
     CI->eraseFromParent();
     return;
@@ -188,8 +191,39 @@ static void scalarizeMaskedLoad(const DataLayout &DL, CallInst *CI,
     return;
   }
 
+  // Optimize the case where the "masked load" is a predicated load - that is,
+  // where the mask is the splat of a non-constant scalar boolean. In that case,
+  // use that splated value as the guard on a conditional vector load.
+  if (isSplatValue(Mask, /*Index=*/0)) {
+    Value *Predicate = Builder.CreateExtractElement(Mask, uint64_t(0ull),
+                                                    Mask->getName() + ".first");
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.load");
+    Builder.SetInsertPoint(CondBlock->getTerminator());
+    LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,
+                                               CI->getName() + ".cond.load");
+    Load->copyMetadata(*CI);
+
+    BasicBlock *PostLoad = ThenTerm->getSuccessor(0);
+    Builder.SetInsertPoint(PostLoad, PostLoad->begin());
+    PHINode *Phi = Builder.CreatePHI(VecType, /*NumReservedValues=*/2);
+    Phi->addIncoming(Load, CondBlock);
+    Phi->addIncoming(Src0, IfBlock);
+    Phi->takeName(CI);
+
+    CI->replaceAllUsesWith(Phi);
+    CI->eraseFromParent();
+    ModifiedDT = true;
+    return;
+  }
   // If the mask is not v1i1, use scalar bit test operations. This generates
   // better results on X86 at least.
+  // Note: this produces worse code on AMDGPU, where the "i1" is implicitly SIMD
+  // - what's a good way to detect this?
   Value *SclrMask;
   if (VectorWidth != 1) {
     Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
@@ -297,7 +331,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, CallInst *CI,
 
   // Short-cut if the mask is all-true.
   if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
-    Builder.CreateAlignedStore(Src, Ptr, AlignVal);
+    StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
+    Store->takeName(CI);
+    Store->copyMetadata(*CI);
     CI->eraseFromParent();
     return;
   }
@@ -319,8 +355,31 @@ static void scalarizeMaskedStore(const DataLayout &DL, CallInst *CI,
     return;
   }
 
+  // Optimize the case where the "masked store" is a predicated store - that is,
+  // when the mask is the splat of a non-constant scalar boolean. In that case,
+  // optimize to a conditional store.
+  if (isSplatValue(Mask, /*Index=*/0)) {
+    Value *Predicate = Builder.CreateExtractElement(Mask, uint64_t(0ull),
+                                                    Mask->getName() + ".first");
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.store");
+    Builder.SetInsertPoint(CondBlock->getTerminator());
+
+    StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
+    Store->takeName(CI);
+    Store->copyMetadata(*CI);
+
+    CI->eraseFromParent();
+    ModifiedDT = true;
+    return;
+  }
+
   // If the mask is not v1i1, use scalar bit test operations. This generates
   // better results on X86 at least.
+
   Value *SclrMask;
   if (VectorWidth != 1) {
     Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
@@ -997,7 +1056,6 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
         any_of(II->args(),
                [](Value *V) { return isa<ScalableVectorType>(V->getType()); }))
       return false;
-
     switch (II->getIntrinsicID()) {
     default:
       break;
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index 30a343b2c564e86..b0da19813f0a4b6 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -19,6 +19,7 @@
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
@@ -49,6 +50,7 @@ STATISTIC(NumNoUndef, "Number of function returns inferred as noundef returns");
 STATISTIC(NumReturnedArg, "Number of arguments inferred as returned");
 STATISTIC(NumWillReturn, "Number of functions inferred as willreturn");
 STATISTIC(NumCold, "Number of functions inferred as cold");
+STATISTIC(NumNoReturn, "Number of functions inferred as no return");
 
 static bool setDoesNotAccessMemory(Function &F) {
   if (F.doesNotAccessMemory())
@@ -66,6 +68,14 @@ static bool setIsCold(Function &F) {
   return true;
 }
 
+static bool setNoReturn(Function &F) {
+  if (F.hasFnAttribute(Attribute::NoReturn))
+    return false;
+  F.addFnAttr(Attribute::NoReturn);
+  ++NumNoReturn;
+  return true;
+}
+
 static bool setOnlyAccessesInaccessibleMemory(Function &F) {
   if (F.onlyAccessesInaccessibleMemory())
     return false;
@@ -1102,6 +1112,15 @@ bool llvm::inferNonMandatoryLibFuncAttrs(Function &F,
   case LibFunc_abort:
     Changed |= setIsCold(F);
     break;
+  case LibFunc_terminate:
+    Changed |= setIsCold(F);
+    Changed |= setNoReturn(F);
+    break;
+  case LibFunc_cxa_throw:
+    Changed |= setIsCold(F);
+    Changed |= setNoReturn(F);
+    // Don't add `nofree` on `__cxa_throw`
+    return Changed;
   // int __nvvm_reflect(const char *)
   case LibFunc_nvvm_reflect:
     Changed |= setRetAndArgsNoUndef(F);
@@ -1963,6 +1982,56 @@ Value *llvm::emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
   return CI;
 }
 
+Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
+                                         const TargetLibraryInfo *TLI,
+                                         LibFunc SizeFeedbackNewFunc,
+                                         uint8_t HotCold) {
+  Module *M = B.GetInsertBlock()->getModule();
+  if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
+    return nullptr;
+
+  StringRef Name = TLI->getName(SizeFeedbackNewFunc);
+
+  // __sized_ptr_t struct return type { void*, size_t }
+  StructType *SizedPtrT =
+      StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
+  FunctionCallee Func =
+      M->getOrInsertFunction(Name, SizedPtrT, Num->getType(), B.getInt8Ty());
+  inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
+  CallInst *CI = B.CreateCall(Func, {Num, B.getInt8(HotCold)}, "sized_ptr");
+
+  if (const Function *F = dyn_cast<Function>(Func.getCallee()))
+    CI->setCallingConv(F->getCallingConv());
+
+  return CI;
+}
+
+Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
+                                                IRBuilderBase &B,
+                                                const TargetLibraryInfo *TLI,
+                                                LibFunc SizeFeedbackNewFunc,
+                                                uint8_t HotCold) {
+  Module *M = B.GetInsertBlock()->getModule();
+  if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
+    return nullptr;
+
+  StringRef Name = TLI->getName(SizeFeedbackNewFunc);
+
+  // __sized_ptr_t struct return type { void*, size_t }
+  StructType *SizedPtrT =
+      StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
+  FunctionCallee Func = M->getOrInsertFunction(Name, SizedPtrT, Num->getType(),
+                                               Align->getType(), B.getInt8Ty());
+  inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
+  CallInst *CI =
+      B.CreateCall(Func, {Num, Align, B.getInt8(HotCold)}, "sized_ptr");
+
+  if (const Function *F = dyn_cast<Function>(Func.getCallee()))
+    CI->setCallingConv(F->getCallingConv());
+
+  return CI;
+}
+
 Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
                             const TargetLibraryInfo *TLI, LibFunc NewFunc,
                             uint8_t HotCold) {
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 37c8761ca9383d0..ebdf760bda7f1ab 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1994,28 +1994,6 @@ static bool canSinkInstructions(
       return false;
   }
 
-  // Because SROA can't handle speculating stores of selects, try not to sink
-  // loads, stores or lifetime markers of allocas when we'd have to create a
-  // PHI for the address operand. Also, because it is likely that loads or
-  // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink
-  // them.
-  // This can cause code churn which can have unintended consequences down
-  // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
-  // FIXME: This is a workaround for a deficiency in SROA - see
-  // https://llvm.org/bugs/show_bug.cgi?id=30188
-  if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) {
-        return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
-      }))
-    return false;
-  if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
-        return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
-      }))
-    return false;
-  if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) {
-        return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
-      }))
-    return false;
-
   // For calls to be sinkable, they must all be indirect, or have same callee.
   // I.e. if we have two direct calls to different callees, we don't want to
   // turn that into an indirect call. Likewise, if we have an indirect call,
@@ -2053,6 +2031,27 @@ static bool canSinkInstructions(
       return I->getOperand(OI) == I0->getOperand(OI);
     };
     if (!all_of(Insts, SameAsI0)) {
+      // Because SROA historically couldn't handle speculating stores of
+      // selects, we try not to sink loads, stores or lifetime markers of
+      // allocas when we'd have to create a PHI for the address operand.
+      // TODO: SROA supports speculation for loads and stores now -- remove
+      // this hack?
+      if (isa<StoreInst>(I0) && OI == 1 &&
+          any_of(Insts, [](const Instruction *I) {
+            return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
+          }))
+        return false;
+      if (isa<LoadInst>(I0) && OI == 0 &&
+          any_of(Insts, [](const Instruction *I) {
+            return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
+          }))
+        return false;
+      if (isLifeTimeMarker(I0) && OI == 1 &&
+          any_of(Insts, [](const Instruction *I) {
+            return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
+          }))
+        return false;
+
       if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) ||
           !canReplaceOperandWithVariable(I0, OI))
         // We can't create a PHI from this GEP.
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 673cc1a6c08f76f..fb2efe581ac6bb2 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/AttributeMask.h"
 #include "llvm/IR/DataLayout.h"
@@ -1745,7 +1746,7 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
   // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
   // Note that in cases where we decide it is "notcold", it might be slightly
   // better to replace the hinted call with a non hinted call, to avoid the
-  // extra paramter and the if condition check of the hint value in the
+  // extra parameter and the if condition check of the hint value in the
   // allocator. This can be considered in the future.
   switch (Func) {
   case LibFunc_Znwm12__hot_cold_t:
@@ -1844,6 +1845,30 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
           TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
           HotCold);
     break;
+  case LibFunc_size_returning_new:
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
+                                         LibFunc_size_returning_new_hot_cold,
+                                         HotCold);
+    break;
+  case LibFunc_size_returning_new_hot_cold:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
+                                         LibFunc_size_returning_new_hot_cold,
+                                         HotCold);
+    break;
+  case LibFunc_size_returning_new_aligned:
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdSizeReturningNewAligned(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_size_returning_new_aligned_hot_cold, HotCold);
+    break;
+  case LibFunc_size_returning_new_aligned_hot_cold:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdSizeReturningNewAligned(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_size_returning_new_aligned_hot_cold, HotCold);
+    break;
   default:
     return nullptr;
   }
@@ -2733,14 +2758,17 @@ Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
     // Note: We don't bother looking any deeper than this first level or for
     // variations of this pattern because instcombine's visitFMUL and/or the
     // reassociation pass should give us this form.
-    Value *OtherMul0, *OtherMul1;
-    if (match(Op0, m_FMul(m_Value(OtherMul0), m_Value(OtherMul1)))) {
-      // Pattern: sqrt((x * y) * z)
-      if (OtherMul0 == OtherMul1 && cast<Instruction>(Op0)->isFast()) {
-        // Matched: sqrt((x * x) * z)
-        RepeatOp = OtherMul0;
-        OtherOp = Op1;
-      }
+    Value *MulOp;
+    if (match(Op0, m_FMul(m_Value(MulOp), m_Deferred(MulOp))) &&
+        cast<Instruction>(Op0)->isFast()) {
+      // Pattern: sqrt((x * x) * z)
+      RepeatOp = MulOp;
+      OtherOp = Op1;
+    } else if (match(Op1, m_FMul(m_Value(MulOp), m_Deferred(MulOp))) &&
+               cast<Instruction>(Op1)->isFast()) {
+      // Pattern: sqrt(z * (x * x))
+      RepeatOp = MulOp;
+      OtherOp = Op0;
     }
   }
   if (!RepeatOp)
@@ -3759,6 +3787,7 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
   Module *M = CI->getModule();
   LibFunc Func;
   Function *Callee = CI->getCalledFunction();
+
   // Check for string/memory library functions.
   if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) {
     // Make sure we never change the calling convention.
@@ -3851,6 +3880,10 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
     case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
     case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
     case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
+    case LibFunc_size_returning_new:
+    case LibFunc_size_returning_new_hot_cold:
+    case LibFunc_size_returning_new_aligned:
+    case LibFunc_size_returning_new_aligned_hot_cold:
       return optimizeNew(CI, Builder, Func);
     default:
       break;
diff --git a/llvm/lib/Transforms/Utils/SizeOpts.cpp b/llvm/lib/Transforms/Utils/SizeOpts.cpp
index 1ca2e0e6ebb9043..09c4c1c3c511ff8 100644
--- a/llvm/lib/Transforms/Utils/SizeOpts.cpp
+++ b/llvm/lib/Transforms/Utils/SizeOpts.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/SizeOpts.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index c35ea431296b704..02ec1d5c259cd61 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -157,6 +157,8 @@ using EqClassKey =
 struct ChainElem {
   Instruction *Inst;
   APInt OffsetFromLeader;
+  ChainElem(Instruction *Inst, APInt OffsetFromLeader)
+      : Inst(std::move(Inst)), OffsetFromLeader(std::move(OffsetFromLeader)) {}
 };
 using Chain = SmallVector<ChainElem, 1>;
 
@@ -187,7 +189,7 @@ constexpr unsigned StackAdjustedAlignment = 4;
 Instruction *propagateMetadata(Instruction *I, const Chain &C) {
   SmallVector<Value *, 8> Values;
   for (const ChainElem &E : C)
-    Values.push_back(E.Inst);
+    Values.emplace_back(E.Inst);
   return propagateMetadata(I, Values);
 }
 
@@ -202,12 +204,12 @@ void reorder(Instruction *I) {
   SmallPtrSet<Instruction *, 16> InstructionsToMove;
   SmallVector<Instruction *, 16> Worklist;
 
-  Worklist.push_back(I);
+  Worklist.emplace_back(I);
   while (!Worklist.empty()) {
     Instruction *IW = Worklist.pop_back_val();
     int NumOperands = IW->getNumOperands();
-    for (int i = 0; i < NumOperands; i++) {
-      Instruction *IM = dyn_cast<Instruction>(IW->getOperand(i));
+    for (int Idx = 0; Idx < NumOperands; Idx++) {
+      Instruction *IM = dyn_cast<Instruction>(IW->getOperand(Idx));
       if (!IM || IM->getOpcode() == Instruction::PHI)
         continue;
 
@@ -220,7 +222,7 @@ void reorder(Instruction *I) {
 
       if (!IM->comesBefore(I)) {
         InstructionsToMove.insert(IM);
-        Worklist.push_back(IM);
+        Worklist.emplace_back(IM);
       }
     }
   }
@@ -228,7 +230,7 @@ void reorder(Instruction *I) {
   // All instructions to move should follow I. Start from I, not from begin().
   for (auto BBI = I->getIterator(), E = I->getParent()->end(); BBI != E;) {
     Instruction *IM = &*(BBI++);
-    if (!InstructionsToMove.count(IM))
+    if (!InstructionsToMove.contains(IM))
       continue;
     IM->moveBefore(I);
   }
@@ -438,11 +440,11 @@ bool Vectorizer::run() {
     assert(!BB->empty());
 
     SmallVector<BasicBlock::iterator, 8> Barriers;
-    Barriers.push_back(BB->begin());
+    Barriers.emplace_back(BB->begin());
     for (Instruction &I : *BB)
       if (!isGuaranteedToTransferExecutionToSuccessor(&I))
-        Barriers.push_back(I.getIterator());
-    Barriers.push_back(BB->end());
+        Barriers.emplace_back(I.getIterator());
+    Barriers.emplace_back(BB->end());
 
     for (auto It = Barriers.begin(), End = std::prev(Barriers.end()); It != End;
          ++It)
@@ -559,14 +561,14 @@ std::vector<Chain> Vectorizer::splitChainByMayAliasInstrs(Chain &C) {
 
     std::vector<Chain> Chains;
     SmallVector<ChainElem, 1> NewChain;
-    NewChain.push_back(*ChainBegin);
+    NewChain.emplace_back(*ChainBegin);
     for (auto ChainIt = std::next(ChainBegin); ChainIt != ChainEnd; ++ChainIt) {
       if (isSafeToMove<IsLoad>(ChainIt->Inst, NewChain.front().Inst,
                                ChainOffsets)) {
         LLVM_DEBUG(dbgs() << "LSV: No intervening may-alias instrs; can merge "
                           << *ChainIt->Inst << " into " << *ChainBegin->Inst
                           << "\n");
-        NewChain.push_back(*ChainIt);
+        NewChain.emplace_back(*ChainIt);
       } else {
         LLVM_DEBUG(
             dbgs() << "LSV: Found intervening may-alias instrs; cannot merge "
@@ -576,7 +578,7 @@ std::vector<Chain> Vectorizer::splitChainByMayAliasInstrs(Chain &C) {
             dbgs() << "LSV: got nontrivial chain without aliasing instrs:\n";
             dumpChain(NewChain);
           });
-          Chains.push_back(std::move(NewChain));
+          Chains.emplace_back(std::move(NewChain));
         }
 
         // Start a new chain.
@@ -588,7 +590,7 @@ std::vector<Chain> Vectorizer::splitChainByMayAliasInstrs(Chain &C) {
         dbgs() << "LSV: got nontrivial chain without aliasing instrs:\n";
         dumpChain(NewChain);
       });
-      Chains.push_back(std::move(NewChain));
+      Chains.emplace_back(std::move(NewChain));
     }
     return Chains;
   };
@@ -689,7 +691,7 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
   });
 
   bool IsLoadChain = isa<LoadInst>(C[0].Inst);
-  auto getVectorFactor = [&](unsigned VF, unsigned LoadStoreSize,
+  auto GetVectorFactor = [&](unsigned VF, unsigned LoadStoreSize,
                              unsigned ChainSizeBytes, VectorType *VecTy) {
     return IsLoadChain ? TTI.getLoadVectorFactor(VF, LoadStoreSize,
                                                  ChainSizeBytes, VecTy)
@@ -721,8 +723,8 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
                  C[CBegin].OffsetFromLeader;
       if (Sz.sgt(VecRegBytes))
         break;
-      CandidateChains.push_back(
-          {CEnd, static_cast<unsigned>(Sz.getLimitedValue())});
+      CandidateChains.emplace_back(CEnd,
+                                   static_cast<unsigned>(Sz.getLimitedValue()));
     }
 
     // Consider the longest chain first.
@@ -746,7 +748,7 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
       unsigned VF = 8 * VecRegBytes / VecElemBits;
 
       // Check that TTI is happy with this vectorization factor.
-      unsigned TargetVF = getVectorFactor(VF, VecElemBits,
+      unsigned TargetVF = GetVectorFactor(VF, VecElemBits,
                                           VecElemBits * NumVecElems / 8, VecTy);
       if (TargetVF != VF && TargetVF < NumVecElems) {
         LLVM_DEBUG(
@@ -840,7 +842,7 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
       // Hooray, we can vectorize this chain!
       Chain &NewChain = Ret.emplace_back();
       for (unsigned I = CBegin; I <= CEnd; ++I)
-        NewChain.push_back(C[I]);
+        NewChain.emplace_back(C[I]);
       CBegin = CEnd; // Skip over the instructions we've added to the chain.
       break;
     }
@@ -959,7 +961,7 @@ bool Vectorizer::vectorizeChain(Chain &C) {
       Vec = Builder.CreateInsertElement(Vec, V, Builder.getInt32(VecIdx++));
     };
     for (const ChainElem &E : C) {
-      auto I = cast<StoreInst>(E.Inst);
+      auto *I = cast<StoreInst>(E.Inst);
       if (FixedVectorType *VT =
               dyn_cast<FixedVectorType>(getLoadStoreType(I))) {
         for (int J = 0, JE = VT->getNumElements(); J < JE; ++J) {
@@ -982,7 +984,7 @@ bool Vectorizer::vectorizeChain(Chain &C) {
   propagateMetadata(VecInst, C);
 
   for (const ChainElem &E : C)
-    ToErase.push_back(E.Inst);
+    ToErase.emplace_back(E.Inst);
 
   ++NumVectorInstructions;
   NumScalarsVectorized += C.size();
@@ -1291,7 +1293,7 @@ std::optional<APInt> Vectorizer::getConstantOffsetSelects(
                         << *ContextInst << ", Depth=" << Depth << "\n");
       std::optional<APInt> TrueDiff = getConstantOffset(
           SelectA->getTrueValue(), SelectB->getTrueValue(), ContextInst, Depth);
-      if (!TrueDiff.has_value())
+      if (!TrueDiff)
         return std::nullopt;
       std::optional<APInt> FalseDiff =
           getConstantOffset(SelectA->getFalseValue(), SelectB->getFalseValue(),
@@ -1308,7 +1310,7 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
                                       BasicBlock::iterator End) {
   EquivalenceClassMap Ret;
 
-  auto getUnderlyingObject = [](const Value *Ptr) -> const Value * {
+  auto GetUnderlyingObject = [](const Value *Ptr) -> const Value * {
     const Value *ObjPtr = llvm::getUnderlyingObject(Ptr);
     if (const auto *Sel = dyn_cast<SelectInst>(ObjPtr)) {
       // The select's themselves are distinct instructions even if they share
@@ -1369,10 +1371,10 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
         (VecTy && TTI.getLoadVectorFactor(VF, TySize, TySize / 8, VecTy) == 0))
       continue;
 
-    Ret[{getUnderlyingObject(Ptr), AS,
+    Ret[{GetUnderlyingObject(Ptr), AS,
          DL.getTypeSizeInBits(getLoadStoreType(&I)->getScalarType()),
          /*IsLoad=*/LI != nullptr}]
-        .push_back(&I);
+        .emplace_back(&I);
   }
 
   return Ret;
@@ -1434,15 +1436,14 @@ std::vector<Chain> Vectorizer::gatherChains(ArrayRef<Instruction *> Instrs) {
     auto ChainIter = MRU.begin();
     for (size_t J = 0; J < MaxChainsToTry && ChainIter != MRU.end();
          ++J, ++ChainIter) {
-      std::optional<APInt> Offset = getConstantOffset(
-          getLoadStorePointerOperand(ChainIter->first),
-          getLoadStorePointerOperand(I),
-          /*ContextInst=*/
-          (ChainIter->first->comesBefore(I) ? I : ChainIter->first));
-      if (Offset.has_value()) {
+      if (std::optional<APInt> Offset = getConstantOffset(
+              getLoadStorePointerOperand(ChainIter->first),
+              getLoadStorePointerOperand(I),
+              /*ContextInst=*/
+              (ChainIter->first->comesBefore(I) ? I : ChainIter->first))) {
         // `Offset` might not have the expected number of bits, if e.g. AS has a
         // different number of bits than opaque pointers.
-        ChainIter->second.push_back(ChainElem{I, Offset.value()});
+        ChainIter->second.emplace_back(I, Offset.value());
         // Move ChainIter to the front of the MRU list.
         MRU.remove(*ChainIter);
         MRU.push_front(*ChainIter);
@@ -1454,7 +1455,7 @@ std::vector<Chain> Vectorizer::gatherChains(ArrayRef<Instruction *> Instrs) {
     if (!MatchFound) {
       APInt ZeroOffset(ASPtrBits, 0);
       InstrListElem *E = new (Allocator.Allocate()) InstrListElem(I);
-      E->second.push_back(ChainElem{I, ZeroOffset});
+      E->second.emplace_back(I, ZeroOffset);
       MRU.push_front(*E);
       Chains.insert(E);
     }
@@ -1465,7 +1466,7 @@ std::vector<Chain> Vectorizer::gatherChains(ArrayRef<Instruction *> Instrs) {
   // Iterate over MRU rather than Chains so the order is deterministic.
   for (auto &E : MRU)
     if (E.second.size() > 1)
-      Ret.push_back(std::move(E.second));
+      Ret.emplace_back(std::move(E.second));
   return Ret;
 }
 
@@ -1509,9 +1510,8 @@ std::optional<APInt> Vectorizer::getConstantOffset(Value *PtrA, Value *PtrB,
       return (OffsetB - OffsetA + Dist).sextOrTrunc(OrigBitWidth);
     }
   }
-  std::optional<APInt> Diff =
-      getConstantOffsetComplexAddrs(PtrA, PtrB, ContextInst, Depth);
-  if (Diff.has_value())
+  if (std::optional<APInt> Diff =
+          getConstantOffsetComplexAddrs(PtrA, PtrB, ContextInst, Depth))
     return (OffsetB - OffsetA + Diff->sext(OffsetB.getBitWidth()))
         .sextOrTrunc(OrigBitWidth);
   return std::nullopt;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0417916546bb109..66a779da8c25bca 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index f62713005393206..7d242082172c62b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -362,12 +362,13 @@ class LoopVectorizationPlanner {
   /// VF and its cost.
   VectorizationFactor planInVPlanNativePath(ElementCount UserVF);
 
-  /// Return the best VPlan for \p VF.
-  VPlan &getBestPlanFor(ElementCount VF) const;
+  /// Return the VPlan for \p VF. At the moment, there is always a single VPlan
+  /// for each VF.
+  VPlan &getPlanFor(ElementCount VF) const;
 
-  /// Return the most profitable vectorization factor. Also collect all
-  /// profitable VFs in ProfitableVFs.
-  ElementCount getBestVF();
+  /// Compute and return the most profitable vectorization factor. Also collect
+  /// all profitable VFs in ProfitableVFs.
+  ElementCount computeBestVF();
 
   /// Generate the IR code for the vectorized loop captured in VPlan \p BestPlan
   /// according to the best selected \p VF and  \p UF.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fdf8f7042c4fb86..d097ae116a0d22d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -380,10 +380,6 @@ cl::opt<bool> llvm::EnableLoopVectorization(
     "vectorize-loops", cl::init(true), cl::Hidden,
     cl::desc("Run the Loop vectorization passes"));
 
-static cl::opt<bool> PrintVPlansInDotFormat(
-    "vplan-print-in-dot-format", cl::Hidden,
-    cl::desc("Use dot format instead of plain text when dumping VPlans"));
-
 static cl::opt<cl::boolOrDefault> ForceSafeDivisor(
     "force-widen-divrem-via-safe-divisor", cl::Hidden,
     cl::desc(
@@ -4354,8 +4350,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
   SmallVector<RecipeVFPair> InvalidCosts;
   for (const auto &Plan : VPlans) {
     for (ElementCount VF : Plan->vectorFactors()) {
-      VPCostContext CostCtx(CM.TTI, Legal->getWidestInductionType(), LLVMCtx,
-                            CM);
+      VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
+                            LLVMCtx, CM);
       auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
       for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
         for (auto &R : *VPBB) {
@@ -6730,9 +6726,12 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
     return RequiresScalarEpilogue &&
            !TheLoop->contains(cast<Instruction>(U)->getParent());
   };
+
+  LoopBlocksDFS DFS(TheLoop);
+  DFS.perform(LI);
   MapVector<Value *, SmallVector<Value *>> DeadInvariantStoreOps;
-  for (BasicBlock *BB : TheLoop->blocks())
-    for (Instruction &I : *BB) {
+  for (BasicBlock *BB : reverse(make_range(DFS.beginRPO(), DFS.endRPO())))
+    for (Instruction &I : reverse(*BB)) {
       // Find all stores to invariant variables. Since they are going to sink
       // outside the loop we do not need calculate cost for them.
       StoreInst *SI;
@@ -6765,6 +6764,13 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
         Value *PointerOp = getLoadStorePointerOperand(&I);
         DeadInterleavePointerOps.push_back(PointerOp);
       }
+
+      // Queue branches for analysis. They are dead, if their successors only
+      // contain dead instructions.
+      if (auto *Br = dyn_cast<BranchInst>(&I)) {
+        if (Br->isConditional())
+          DeadOps.push_back(&I);
+      }
     }
 
   // Mark ops feeding interleave group members as free, if they are only used
@@ -6789,8 +6795,36 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
   // Mark ops that would be trivially dead and are only used by ignored
   // instructions as free.
   BasicBlock *Header = TheLoop->getHeader();
+
+  // Returns true if the block contains only dead instructions. Such blocks will
+  // be removed by VPlan-to-VPlan transforms and won't be considered by the
+  // VPlan-based cost model, so skip them in the legacy cost-model as well.
+  auto IsEmptyBlock = [this](BasicBlock *BB) {
+    return all_of(*BB, [this](Instruction &I) {
+      return ValuesToIgnore.contains(&I) || VecValuesToIgnore.contains(&I) ||
+             (isa<BranchInst>(&I) && !cast<BranchInst>(&I)->isConditional());
+    });
+  };
   for (unsigned I = 0; I != DeadOps.size(); ++I) {
     auto *Op = dyn_cast<Instruction>(DeadOps[I]);
+
+    // Check if the branch should be considered dead.
+    if (auto *Br = dyn_cast_or_null<BranchInst>(Op)) {
+      BasicBlock *ThenBB = Br->getSuccessor(0);
+      BasicBlock *ElseBB = Br->getSuccessor(1);
+      bool ThenEmpty = IsEmptyBlock(ThenBB);
+      bool ElseEmpty = IsEmptyBlock(ElseBB);
+      if ((ThenEmpty && ElseEmpty) ||
+          (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
+           ElseBB->phis().empty()) ||
+          (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
+           ThenBB->phis().empty())) {
+        VecValuesToIgnore.insert(Br);
+        DeadOps.push_back(Br->getCondition());
+      }
+      continue;
+    }
+
     // Skip any op that shouldn't be considered dead.
     if (!Op || !TheLoop->contains(Op) ||
         (isa<PHINode>(Op) && Op->getParent() == Header) ||
@@ -6980,27 +7014,32 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
 
   ElementCount MaxUserVF =
       UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
-  bool UserVFIsLegal = ElementCount::isKnownLE(UserVF, MaxUserVF);
-  if (!UserVF.isZero() && UserVFIsLegal) {
-    assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
-           "VF needs to be a power of two");
-    // Collect the instructions (and their associated costs) that will be more
-    // profitable to scalarize.
-    CM.collectInLoopReductions();
-    if (CM.selectUserVectorizationFactor(UserVF)) {
-      LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
-      buildVPlansWithVPRecipes(UserVF, UserVF);
-      if (!hasPlanWithVF(UserVF)) {
-        LLVM_DEBUG(dbgs() << "LV: No VPlan could be built for " << UserVF
-                          << ".\n");
-        return std::nullopt;
-      }
+  if (!UserVF.isZero()) {
+    if (!ElementCount::isKnownLE(UserVF, MaxUserVF)) {
+      reportVectorizationInfo(
+          "UserVF ignored because it may be larger than the maximal safe VF",
+          "InvalidUserVF", ORE, OrigLoop);
+    } else {
+      assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
+             "VF needs to be a power of two");
+      // Collect the instructions (and their associated costs) that will be more
+      // profitable to scalarize.
+      CM.collectInLoopReductions();
+      if (CM.selectUserVectorizationFactor(UserVF)) {
+        LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
+        buildVPlansWithVPRecipes(UserVF, UserVF);
+        if (!hasPlanWithVF(UserVF)) {
+          LLVM_DEBUG(dbgs()
+                     << "LV: No VPlan could be built for " << UserVF << ".\n");
+          return std::nullopt;
+        }
 
-      LLVM_DEBUG(printPlans(dbgs()));
-      return {{UserVF, 0, 0}};
-    } else
-      reportVectorizationInfo("UserVF ignored because of invalid costs.",
-                              "InvalidCost", ORE, OrigLoop);
+        LLVM_DEBUG(printPlans(dbgs()));
+        return {{UserVF, 0, 0}};
+      } else
+        reportVectorizationInfo("UserVF ignored because of invalid costs.",
+                                "InvalidCost", ORE, OrigLoop);
+    }
   }
 
   // Collect the Vectorization Factor Candidates.
@@ -7062,7 +7101,8 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
                                                ElementCount VF) const {
   InstructionCost Cost = 0;
   LLVMContext &LLVMCtx = OrigLoop->getHeader()->getContext();
-  VPCostContext CostCtx(CM.TTI, Legal->getWidestInductionType(), LLVMCtx, CM);
+  VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
+                        LLVMCtx, CM);
 
   // Cost modeling for inductions is inaccurate in the legacy cost model
   // compared to the recipes that are generated. To match here initially during
@@ -7215,7 +7255,7 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
   return Cost;
 }
 
-ElementCount LoopVectorizationPlanner::getBestVF() {
+ElementCount LoopVectorizationPlanner::computeBestVF() {
   // If there is a single VPlan with a single VF, return it directly.
   VPlan &FirstPlan = *VPlans[0];
   if (VPlans.size() == 1 && size(FirstPlan.vectorFactors()) == 1)
@@ -7263,19 +7303,6 @@ ElementCount LoopVectorizationPlanner::getBestVF() {
   return BestFactor.Width;
 }
 
-VPlan &LoopVectorizationPlanner::getBestPlanFor(ElementCount VF) const {
-  assert(count_if(VPlans,
-                  [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
-             1 &&
-         "Best VF has not a single VPlan.");
-
-  for (const VPlanPtr &Plan : VPlans) {
-    if (Plan->hasVF(VF))
-      return *Plan.get();
-  }
-  llvm_unreachable("No plan found!");
-}
-
 static void AddRuntimeUnrollDisableMetaData(Loop *L) {
   SmallVector<Metadata *, 4> MDs;
   // Reserve first location for self reference to the LoopID metadata node.
@@ -7520,16 +7547,6 @@ LoopVectorizationPlanner::executePlan(
   return {State.ExpandedSCEVs, ReductionResumeValues};
 }
 
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void LoopVectorizationPlanner::printPlans(raw_ostream &O) {
-  for (const auto &Plan : VPlans)
-    if (PrintVPlansInDotFormat)
-      Plan->printDOT(O);
-    else
-      Plan->print(O);
-}
-#endif
-
 //===--------------------------------------------------------------------===//
 // EpilogueVectorizerMainLoop
 //===--------------------------------------------------------------------===//
@@ -7819,35 +7836,6 @@ void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
   });
 }
 
-bool LoopVectorizationPlanner::getDecisionAndClampRange(
-    const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
-  assert(!Range.isEmpty() && "Trying to test an empty VF range.");
-  bool PredicateAtRangeStart = Predicate(Range.Start);
-
-  for (ElementCount TmpVF : VFRange(Range.Start * 2, Range.End))
-    if (Predicate(TmpVF) != PredicateAtRangeStart) {
-      Range.End = TmpVF;
-      break;
-    }
-
-  return PredicateAtRangeStart;
-}
-
-/// Build VPlans for the full range of feasible VF's = {\p MinVF, 2 * \p MinVF,
-/// 4 * \p MinVF, ..., \p MaxVF} by repeatedly building a VPlan for a sub-range
-/// of VF's starting at a given VF and extending it as much as possible. Each
-/// vectorization decision can potentially shorten this sub-range during
-/// buildVPlan().
-void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
-                                           ElementCount MaxVF) {
-  auto MaxVFTimes2 = MaxVF * 2;
-  for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
-    VFRange SubRange = {VF, MaxVFTimes2};
-    VPlans.push_back(buildVPlan(SubRange));
-    VF = SubRange.End;
-  }
-}
-
 iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
 VPRecipeBuilder::mapToVPValues(User::op_range Operands) {
   std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
@@ -9499,7 +9487,7 @@ static bool processLoopInVPlanNativePath(
   if (VPlanBuildStressTest || VectorizationFactor::Disabled() == VF)
     return false;
 
-  VPlan &BestPlan = LVP.getBestPlanFor(VF.Width);
+  VPlan &BestPlan = LVP.getPlanFor(VF.Width);
 
   {
     bool AddBranchWeights =
@@ -9976,10 +9964,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL,
                                  &CM, BFI, PSI, Checks);
 
-      ElementCount BestVF = LVP.getBestVF();
+      ElementCount BestVF = LVP.computeBestVF();
       assert(BestVF.isScalar() &&
              "VPlan cost model and legacy cost model disagreed");
-      VPlan &BestPlan = LVP.getBestPlanFor(BestVF);
+      VPlan &BestPlan = LVP.getPlanFor(BestVF);
       LVP.executePlan(BestVF, IC, BestPlan, Unroller, DT, false);
 
       ORE->emit([&]() {
@@ -9991,11 +9979,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     } else {
       // If we decided that it is *legal* to vectorize the loop, then do it.
 
-      ElementCount BestVF = LVP.getBestVF();
+      ElementCount BestVF = LVP.computeBestVF();
       LLVM_DEBUG(dbgs() << "VF picked by VPlan cost model: " << BestVF << "\n");
       assert(VF.Width == BestVF &&
              "VPlan cost model and legacy cost model disagreed");
-      VPlan &BestPlan = LVP.getBestPlanFor(BestVF);
+      VPlan &BestPlan = LVP.getPlanFor(BestVF);
       // Consider vectorizing the epilogue too if it's profitable.
       VectorizationFactor EpilogueVF =
           LVP.selectEpilogueVectorizationFactor(BestVF, IC);
@@ -10021,7 +10009,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
                                                  ORE, EPI, &LVL, &CM, BFI, PSI,
                                                  Checks);
 
-        VPlan &BestEpiPlan = LVP.getBestPlanFor(EPI.EpilogueVF);
+        VPlan &BestEpiPlan = LVP.getPlanFor(EPI.EpilogueVF);
         VPRegionBlock *VectorLoop = BestEpiPlan.getVectorLoopRegion();
         VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
         Header->setName("vec.epilog.vector.body");
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 81c4193cfe1081e..dee6d688b1b9032 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -795,16 +795,6 @@ struct InstructionsState {
 
 } // end anonymous namespace
 
-/// Chooses the correct key for scheduling data. If \p Op has the same (or
-/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
-/// OpValue.
-static Value *isOneOf(const InstructionsState &S, Value *Op) {
-  auto *I = dyn_cast<Instruction>(Op);
-  if (I && S.isOpcodeOrAlt(I))
-    return Op;
-  return S.OpValue;
-}
-
 /// \returns true if \p Opcode is allowed as part of the main/alternate
 /// instruction for SLP vectorization.
 ///
@@ -1243,11 +1233,13 @@ class BoUpSLP {
     StridedVectorize
   };
 
-  using ValueList = SmallVector<Value *, 4>;
-  using ValueSet = SmallPtrSet<Value *, 8>;
+  using ValueList = SmallVector<Value *, 8>;
+  using InstrList = SmallVector<Instruction *, 16>;
+  using ValueSet = SmallPtrSet<Value *, 16>;
+  using StoreList = SmallVector<StoreInst *, 8>;
   using ExtraValueToDebugLocsMap =
       MapVector<Value *, SmallVector<Instruction *, 2>>;
-  using OrdersType = SmallVector<unsigned, 0>;
+  using OrdersType = SmallVector<unsigned, 4>;
 
   BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
           TargetLibraryInfo *TLi, AAResults *Aa, LoopInfo *Li,
@@ -1469,7 +1461,7 @@ class BoUpSLP {
   /// \param TryRecursiveCheck used to check if long masked gather can be
   /// represented as a serie of loads/insert subvector, if profitable.
   LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
-                               OrdersType &Order,
+                               SmallVectorImpl<unsigned> &Order,
                                SmallVectorImpl<Value *> &PointerOps,
                                bool TryRecursiveCheck = true) const;
 
@@ -2838,7 +2830,7 @@ class BoUpSLP {
   /// \param ResizeAllowed indicates whether it is allowed to handle subvector
   /// extract order.
   bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
-                       OrdersType &CurrentOrder,
+                       SmallVectorImpl<unsigned> &CurrentOrder,
                        bool ResizeAllowed = false) const;
 
   /// Vectorize a single entry in the tree.
@@ -3082,10 +3074,10 @@ class BoUpSLP {
     CombinedOpcode CombinedOp = NotCombinedOp;
 
     /// Does this sequence require some shuffling?
-    SmallVector<int, 0> ReuseShuffleIndices;
+    SmallVector<int, 4> ReuseShuffleIndices;
 
     /// Does this entry require reordering?
-    OrdersType ReorderIndices;
+    SmallVector<unsigned, 4> ReorderIndices;
 
     /// Points back to the VectorizableTree.
     ///
@@ -3106,7 +3098,7 @@ class BoUpSLP {
     /// The operands of each instruction in each lane Operands[op_index][lane].
     /// Note: This helps avoid the replication of the code that performs the
     /// reordering of operands during buildTree_rec() and vectorizeTree().
-    SmallVector<ValueList, 0> Operands;
+    SmallVector<ValueList, 2> Operands;
 
     /// The main/alternate instruction.
     Instruction *MainOp = nullptr;
@@ -3581,14 +3573,14 @@ class BoUpSLP {
 
     ScheduleData() = default;
 
-    void init(int BlockSchedulingRegionID, Value *OpVal) {
+    void init(int BlockSchedulingRegionID, Instruction *I) {
       FirstInBundle = this;
       NextInBundle = nullptr;
       NextLoadStore = nullptr;
       IsScheduled = false;
       SchedulingRegionID = BlockSchedulingRegionID;
       clearDependencies();
-      OpValue = OpVal;
+      Inst = I;
       TE = nullptr;
     }
 
@@ -3694,9 +3686,6 @@ class BoUpSLP {
 
     Instruction *Inst = nullptr;
 
-    /// Opcode of the current instruction in the schedule data.
-    Value *OpValue = nullptr;
-
     /// The TreeEntry that this instruction corresponds to.
     TreeEntry *TE = nullptr;
 
@@ -3714,13 +3703,13 @@ class BoUpSLP {
 
     /// The dependent memory instructions.
     /// This list is derived on demand in calculateDependencies().
-    SmallVector<ScheduleData *, 0> MemoryDependencies;
+    SmallVector<ScheduleData *, 4> MemoryDependencies;
 
     /// List of instructions which this instruction could be control dependent
     /// on.  Allowing such nodes to be scheduled below this one could introduce
     /// a runtime fault which didn't exist in the original program.
     /// ex: this is a load or udiv following a readonly call which inf loops
-    SmallVector<ScheduleData *, 0> ControlDependencies;
+    SmallVector<ScheduleData *, 4> ControlDependencies;
 
     /// This ScheduleData is in the current scheduling region if this matches
     /// the current SchedulingRegionID of BlockScheduling.
@@ -3813,18 +3802,6 @@ class BoUpSLP {
       return nullptr;
     }
 
-    ScheduleData *getScheduleData(Value *V, Value *Key) {
-      if (V == Key)
-        return getScheduleData(V);
-      auto I = ExtraScheduleDataMap.find(V);
-      if (I != ExtraScheduleDataMap.end()) {
-        ScheduleData *SD = I->second.lookup(Key);
-        if (SD && isInSchedulingRegion(SD))
-          return SD;
-      }
-      return nullptr;
-    }
-
     bool isInSchedulingRegion(ScheduleData *SD) const {
       return SD->SchedulingRegionID == SchedulingRegionID;
     }
@@ -3838,27 +3815,24 @@ class BoUpSLP {
 
       for (ScheduleData *BundleMember = SD; BundleMember;
            BundleMember = BundleMember->NextInBundle) {
-        if (BundleMember->Inst != BundleMember->OpValue)
-          continue;
 
         // Handle the def-use chain dependencies.
 
         // Decrement the unscheduled counter and insert to ready list if ready.
         auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
-          doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
-            if (OpDef && OpDef->hasValidDependencies() &&
-                OpDef->incrementUnscheduledDeps(-1) == 0) {
-              // There are no more unscheduled dependencies after
-              // decrementing, so we can put the dependent instruction
-              // into the ready list.
-              ScheduleData *DepBundle = OpDef->FirstInBundle;
-              assert(!DepBundle->IsScheduled &&
-                     "already scheduled bundle gets ready");
-              ReadyList.insert(DepBundle);
-              LLVM_DEBUG(dbgs()
-                         << "SLP:    gets ready (def): " << *DepBundle << "\n");
-            }
-          });
+          ScheduleData *OpDef = getScheduleData(I);
+          if (OpDef && OpDef->hasValidDependencies() &&
+              OpDef->incrementUnscheduledDeps(-1) == 0) {
+            // There are no more unscheduled dependencies after
+            // decrementing, so we can put the dependent instruction
+            // into the ready list.
+            ScheduleData *DepBundle = OpDef->FirstInBundle;
+            assert(!DepBundle->IsScheduled &&
+                   "already scheduled bundle gets ready");
+            ReadyList.insert(DepBundle);
+            LLVM_DEBUG(dbgs()
+                       << "SLP:    gets ready (def): " << *DepBundle << "\n");
+          }
         };
 
         // If BundleMember is a vector bundle, its operands may have been
@@ -3942,8 +3916,7 @@ class BoUpSLP {
                "primary schedule data not in window?");
         assert(isInSchedulingRegion(SD->FirstInBundle) &&
                "entire bundle in window!");
-        (void)SD;
-        doForAllOpcodes(I, [](ScheduleData *SD) { SD->verify(); });
+        SD->verify();
       }
 
       for (auto *SD : ReadyInsts) {
@@ -3953,29 +3926,17 @@ class BoUpSLP {
       }
     }
 
-    void doForAllOpcodes(Value *V,
-                         function_ref<void(ScheduleData *SD)> Action) {
-      if (ScheduleData *SD = getScheduleData(V))
-        Action(SD);
-      auto I = ExtraScheduleDataMap.find(V);
-      if (I != ExtraScheduleDataMap.end())
-        for (auto &P : I->second)
-          if (isInSchedulingRegion(P.second))
-            Action(P.second);
-    }
-
     /// Put all instructions into the ReadyList which are ready for scheduling.
     template <typename ReadyListType>
     void initialFillReadyList(ReadyListType &ReadyList) {
       for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
-        doForAllOpcodes(I, [&](ScheduleData *SD) {
-          if (SD->isSchedulingEntity() && SD->hasValidDependencies() &&
-              SD->isReady()) {
-            ReadyList.insert(SD);
-            LLVM_DEBUG(dbgs()
-                       << "SLP:    initially in ready list: " << *SD << "\n");
-          }
-        });
+        ScheduleData *SD = getScheduleData(I);
+        if (SD && SD->isSchedulingEntity() && SD->hasValidDependencies() &&
+            SD->isReady()) {
+          ReadyList.insert(SD);
+          LLVM_DEBUG(dbgs()
+                     << "SLP:    initially in ready list: " << *SD << "\n");
+        }
       }
     }
 
@@ -4033,10 +3994,6 @@ class BoUpSLP {
     /// ScheduleData structures are recycled.
     DenseMap<Instruction *, ScheduleData *> ScheduleDataMap;
 
-    /// Attaches ScheduleData to Instruction with the leading key.
-    DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
-        ExtraScheduleDataMap;
-
     /// The ready-list for scheduling (only used for the dry-run).
     SetVector<ScheduleData *> ReadyInsts;
 
@@ -4298,12 +4255,12 @@ static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
 /// the original order of the scalars. Procedure transforms the provided order
 /// in accordance with the given \p Mask. If the resulting \p Order is just an
 /// identity order, \p Order is cleared.
-static void reorderOrder(BoUpSLP::OrdersType &Order, ArrayRef<int> Mask,
+static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
                          bool BottomOrder = false) {
   assert(!Mask.empty() && "Expected non-empty mask.");
   unsigned Sz = Mask.size();
   if (BottomOrder) {
-    BoUpSLP::OrdersType PrevOrder;
+    SmallVector<unsigned> PrevOrder;
     if (Order.empty()) {
       PrevOrder.resize(Sz);
       std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
@@ -4693,7 +4650,7 @@ getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
 }
 
 BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
-    ArrayRef<Value *> VL, const Value *VL0, OrdersType &Order,
+    ArrayRef<Value *> VL, const Value *VL0, SmallVectorImpl<unsigned> &Order,
     SmallVectorImpl<Value *> &PointerOps, bool TryRecursiveCheck) const {
   // Check that a vectorized load would load the same memory as a scalar
   // load. For example, we don't want to vectorize loads that are smaller
@@ -4821,7 +4778,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
         for (unsigned Cnt = 0, End = VL.size(); Cnt + VF <= End;
              Cnt += VF, ++VectorizedCnt) {
           ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
-          OrdersType Order;
+          SmallVector<unsigned> Order;
           SmallVector<Value *> PointerOps;
           LoadsState LS =
               canVectorizeLoads(Slice, Slice.front(), Order, PointerOps,
@@ -5395,7 +5352,7 @@ void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
   TE.ReorderIndices.clear();
   // Try to improve gathered nodes with clustered reuses, if possible.
   ArrayRef<int> Slice = ArrayRef(NewMask).slice(0, Sz);
-  OrdersType NewOrder(Slice);
+  SmallVector<unsigned> NewOrder(Slice);
   inversePermutation(NewOrder, NewMask);
   reorderScalars(TE.Scalars, NewMask);
   // Fill the reuses mask with the identity submasks.
@@ -7142,6 +7099,21 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                        ReuseShuffleIndices);
         return;
       }
+      SmallPtrSet<const TreeEntry *, 4> Nodes;
+      Nodes.insert(getTreeEntry(S.OpValue));
+      for (const TreeEntry *E : MultiNodeScalars.lookup(S.OpValue))
+        Nodes.insert(E);
+      SmallPtrSet<Value *, 8> Values(VL.begin(), VL.end());
+      if (any_of(Nodes, [&](const TreeEntry *E) {
+            return all_of(E->Scalars,
+                          [&](Value *V) { return Values.contains(V); });
+          })) {
+        LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
+        if (TryToFindDuplicates(S))
+          newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+                       ReuseShuffleIndices);
+        return;
+      }
     } else {
       // Record the reuse of the tree node.  FIXME, currently this is only used
       // to properly draw the graph rather than for the actual vectorization.
@@ -7715,7 +7687,7 @@ unsigned BoUpSLP::canMapToVector(Type *T) const {
 }
 
 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
-                              OrdersType &CurrentOrder,
+                              SmallVectorImpl<unsigned> &CurrentOrder,
                               bool ResizeAllowed) const {
   const auto *It = find_if(VL, IsaPred<ExtractElementInst, ExtractValueInst>);
   assert(It != VL.end() && "Expected at least one extract instruction.");
@@ -11881,8 +11853,7 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
     auto *Bundle = BlocksSchedules[BB]->getScheduleData(V);
     if (Bundle && Bundle->isPartOfBundle())
       for (; Bundle; Bundle = Bundle->NextInBundle)
-        if (Bundle->OpValue == Bundle->Inst)
-          Res.second = Bundle->Inst;
+        Res.second = Bundle->Inst;
   }
 
   // LastInst can still be null at this point if there's either not an entry
@@ -11975,7 +11946,7 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
     if (auto *VecTy = dyn_cast<FixedVectorType>(Scalar->getType())) {
       assert(SLPReVec && "FixedVectorType is not expected.");
       Vec = InsElt = Builder.CreateInsertVector(
-          Vec->getType(), Vec, V,
+          Vec->getType(), Vec, Scalar,
           Builder.getInt64(Pos * VecTy->getNumElements()));
       auto *II = dyn_cast<IntrinsicInst>(InsElt);
       if (!II || II->getIntrinsicID() != Intrinsic::vector_insert)
@@ -14949,7 +14920,8 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
     // initial bundle to the region.
     if (ScheduleEnd != OldScheduleEnd) {
       for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode())
-        doForAllOpcodes(I, [](ScheduleData *SD) { SD->clearDependencies(); });
+        if (ScheduleData *SD = getScheduleData(I))
+          SD->clearDependencies();
       ReSchedule = true;
     }
     if (Bundle) {
@@ -15068,37 +15040,21 @@ BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
   return &(ScheduleDataChunks.back()[ChunkPos++]);
 }
 
-bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
-                                                      const InstructionsState &S) {
-  if (getScheduleData(V, isOneOf(S, V)))
-    return true;
+bool BoUpSLP::BlockScheduling::extendSchedulingRegion(
+    Value *V, const InstructionsState &S) {
   Instruction *I = dyn_cast<Instruction>(V);
   assert(I && "bundle member must be an instruction");
   assert(!isa<PHINode>(I) && !isVectorLikeInstWithConstOps(I) &&
          !doesNotNeedToBeScheduled(I) &&
          "phi nodes/insertelements/extractelements/extractvalues don't need to "
          "be scheduled");
-  auto &&CheckScheduleForI = [this, &S](Instruction *I) -> bool {
-    ScheduleData *ISD = getScheduleData(I);
-    if (!ISD)
-      return false;
-    assert(isInSchedulingRegion(ISD) &&
-           "ScheduleData not in scheduling region");
-    ScheduleData *SD = allocateScheduleDataChunks();
-    SD->Inst = I;
-    SD->init(SchedulingRegionID, S.OpValue);
-    ExtraScheduleDataMap[I][S.OpValue] = SD;
-    return true;
-  };
-  if (CheckScheduleForI(I))
+  if (getScheduleData(I))
     return true;
   if (!ScheduleStart) {
     // It's the first instruction in the new region.
     initScheduleData(I, I->getNextNode(), nullptr, nullptr);
     ScheduleStart = I;
     ScheduleEnd = I->getNextNode();
-    if (isOneOf(S, I) != I)
-      CheckScheduleForI(I);
     assert(ScheduleEnd && "tried to vectorize a terminator?");
     LLVM_DEBUG(dbgs() << "SLP:  initialize schedule region to " << *I << "\n");
     return true;
@@ -15137,8 +15093,6 @@ bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
            "Instruction is in wrong basic block.");
     initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
     ScheduleStart = I;
-    if (isOneOf(S, I) != I)
-      CheckScheduleForI(I);
     LLVM_DEBUG(dbgs() << "SLP:  extend schedule region start to " << *I
                       << "\n");
     return true;
@@ -15151,8 +15105,6 @@ bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
   initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
                    nullptr);
   ScheduleEnd = I->getNextNode();
-  if (isOneOf(S, I) != I)
-    CheckScheduleForI(I);
   assert(ScheduleEnd && "tried to vectorize a terminator?");
   LLVM_DEBUG(dbgs() << "SLP:  extend schedule region end to " << *I << "\n");
   return true;
@@ -15171,7 +15123,6 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
     if (!SD) {
       SD = allocateScheduleDataChunks();
       ScheduleDataMap[I] = SD;
-      SD->Inst = I;
     }
     assert(!isInSchedulingRegion(SD) &&
            "new ScheduleData already in scheduling region");
@@ -15225,8 +15176,8 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
       BundleMember->resetUnscheduledDeps();
 
       // Handle def-use chain dependencies.
-      if (BundleMember->OpValue != BundleMember->Inst) {
-        if (ScheduleData *UseSD = getScheduleData(BundleMember->Inst)) {
+      for (User *U : BundleMember->Inst->users()) {
+        if (ScheduleData *UseSD = getScheduleData(cast<Instruction>(U))) {
           BundleMember->Dependencies++;
           ScheduleData *DestBundle = UseSD->FirstInBundle;
           if (!DestBundle->IsScheduled)
@@ -15234,17 +15185,6 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
           if (!DestBundle->hasValidDependencies())
             WorkList.push_back(DestBundle);
         }
-      } else {
-        for (User *U : BundleMember->Inst->users()) {
-          if (ScheduleData *UseSD = getScheduleData(cast<Instruction>(U))) {
-            BundleMember->Dependencies++;
-            ScheduleData *DestBundle = UseSD->FirstInBundle;
-            if (!DestBundle->IsScheduled)
-              BundleMember->incrementUnscheduledDeps(1);
-            if (!DestBundle->hasValidDependencies())
-              WorkList.push_back(DestBundle);
-          }
-        }
       }
 
       auto MakeControlDependent = [&](Instruction *I) {
@@ -15392,12 +15332,12 @@ void BoUpSLP::BlockScheduling::resetSchedule() {
   assert(ScheduleStart &&
          "tried to reset schedule on block which has not been scheduled");
   for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
-    doForAllOpcodes(I, [&](ScheduleData *SD) {
+    if (ScheduleData *SD = getScheduleData(I)) {
       assert(isInSchedulingRegion(SD) &&
              "ScheduleData not in scheduling region");
       SD->IsScheduled = false;
       SD->resetUnscheduledDeps();
-    });
+    }
   }
   ReadyInsts.clear();
 }
@@ -15432,7 +15372,7 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
   int Idx = 0;
   for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
        I = I->getNextNode()) {
-    BS->doForAllOpcodes(I, [this, &Idx, BS](ScheduleData *SD) {
+    if (ScheduleData *SD = BS->getScheduleData(I)) {
       TreeEntry *SDTE = getTreeEntry(SD->Inst);
       (void)SDTE;
       assert((isVectorLikeInstWithConstOps(SD->Inst) ||
@@ -15443,7 +15383,7 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
 
       if (SD->isSchedulingEntity() && SD->isPartOfBundle())
         BS->calculateDependencies(SD, false, this);
-    });
+    }
   }
   BS->initialFillReadyList(ReadyInsts);
 
@@ -15475,11 +15415,9 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
 #if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
   // Check that all schedulable entities got scheduled
   for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; I = I->getNextNode()) {
-    BS->doForAllOpcodes(I, [&](ScheduleData *SD) {
-      if (SD->isSchedulingEntity() && SD->hasValidDependencies()) {
-        assert(SD->IsScheduled && "must be scheduled at this point");
-      }
-    });
+    ScheduleData *SD = BS->getScheduleData(I);
+    if (SD && SD->isSchedulingEntity() && SD->hasValidDependencies())
+      assert(SD->IsScheduled && "must be scheduled at this point");
   }
 #endif
 
@@ -15661,7 +15599,8 @@ bool BoUpSLP::collectValuesToDemote(
   if (any_of(E.Scalars, [&](Value *V) {
         return !all_of(V->users(), [=](User *U) {
           return getTreeEntry(U) ||
-                 (UserIgnoreList && UserIgnoreList->contains(U)) ||
+                 (E.Idx == 0 && UserIgnoreList &&
+                  UserIgnoreList->contains(U)) ||
                  (!isa<CmpInst>(U) && U->getType()->isSized() &&
                   !U->getType()->isScalableTy() &&
                   DL->getTypeSizeInBits(U->getType()) <= BitWidth);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2a9e4e12190cdc2..15acfc00251e291 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -21,6 +21,7 @@
 #include "VPlanCFG.h"
 #include "VPlanDominatorTree.h"
 #include "VPlanPatternMatch.h"
+#include "VPlanTransforms.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
@@ -55,6 +56,10 @@ namespace llvm {
 extern cl::opt<bool> EnableVPlanNativePath;
 }
 
+static cl::opt<bool> PrintVPlansInDotFormat(
+    "vplan-print-in-dot-format", cl::Hidden,
+    cl::desc("Use dot format instead of plain text when dumping VPlans"));
+
 #define DEBUG_TYPE "vplan"
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1643,3 +1648,57 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) {
   return match(V, m_Binary<Instruction::ICmp>(m_VPValue(A), m_VPValue(B))) &&
          IsWideCanonicalIV(A) && B == Plan.getOrCreateBackedgeTakenCount();
 }
+
+bool LoopVectorizationPlanner::getDecisionAndClampRange(
+    const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
+  assert(!Range.isEmpty() && "Trying to test an empty VF range.");
+  bool PredicateAtRangeStart = Predicate(Range.Start);
+
+  for (ElementCount TmpVF : VFRange(Range.Start * 2, Range.End))
+    if (Predicate(TmpVF) != PredicateAtRangeStart) {
+      Range.End = TmpVF;
+      break;
+    }
+
+  return PredicateAtRangeStart;
+}
+
+/// Build VPlans for the full range of feasible VF's = {\p MinVF, 2 * \p MinVF,
+/// 4 * \p MinVF, ..., \p MaxVF} by repeatedly building a VPlan for a sub-range
+/// of VF's starting at a given VF and extending it as much as possible. Each
+/// vectorization decision can potentially shorten this sub-range during
+/// buildVPlan().
+void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
+                                           ElementCount MaxVF) {
+  auto MaxVFTimes2 = MaxVF * 2;
+  for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
+    VFRange SubRange = {VF, MaxVFTimes2};
+    auto Plan = buildVPlan(SubRange);
+    VPlanTransforms::optimize(*Plan, *PSE.getSE());
+    VPlans.push_back(std::move(Plan));
+    VF = SubRange.End;
+  }
+}
+
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
+  assert(count_if(VPlans,
+                  [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
+             1 &&
+         "Multiple VPlans for VF.");
+
+  for (const VPlanPtr &Plan : VPlans) {
+    if (Plan->hasVF(VF))
+      return *Plan.get();
+  }
+  llvm_unreachable("No plan found!");
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void LoopVectorizationPlanner::printPlans(raw_ostream &O) {
+  for (const auto &Plan : VPlans)
+    if (PrintVPlansInDotFormat)
+      Plan->printDOT(O);
+    else
+      Plan->print(O);
+}
+#endif
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 016ad75c21ceb07..a99f3882092c2c9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -736,14 +736,16 @@ class VPLiveOut : public VPUser {
 /// Struct to hold various analysis needed for cost computations.
 struct VPCostContext {
   const TargetTransformInfo &TTI;
+  const TargetLibraryInfo &TLI;
   VPTypeAnalysis Types;
   LLVMContext &LLVMCtx;
   LoopVectorizationCostModel &CM;
   SmallPtrSet<Instruction *, 8> SkipCostComputation;
 
-  VPCostContext(const TargetTransformInfo &TTI, Type *CanIVTy,
-                LLVMContext &LLVMCtx, LoopVectorizationCostModel &CM)
-      : TTI(TTI), Types(CanIVTy, LLVMCtx), LLVMCtx(LLVMCtx), CM(CM) {}
+  VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
+                Type *CanIVTy, LLVMContext &LLVMCtx,
+                LoopVectorizationCostModel &CM)
+      : TTI(TTI), TLI(TLI), Types(CanIVTy, LLVMCtx), LLVMCtx(LLVMCtx), CM(CM) {}
 
   /// Return the cost for \p UI with \p VF using the legacy cost model as
   /// fallback until computing the cost of all recipes migrates to VPlan.
@@ -796,7 +798,7 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
   /// Return the cost of this recipe, taking into account if the cost
   /// computation should be skipped and the ForceTargetInstructionCost flag.
   /// Also takes care of printing the cost for debugging.
-  virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
+  InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
 
   /// Insert an unlinked recipe into a basic block immediately before
   /// the specified recipe.
@@ -860,9 +862,11 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
   DebugLoc getDebugLoc() const { return DL; }
 
 protected:
-  /// Compute the cost of this recipe using the legacy cost model and the
-  /// underlying instructions.
-  InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const;
+  /// Compute the cost of this recipe either using a recipe's specialized
+  /// implementation or using the legacy cost model and the underlying
+  /// instructions.
+  virtual InstructionCost computeCost(ElementCount VF,
+                                      VPCostContext &Ctx) const;
 };
 
 // Helper macro to define common classof implementations for recipes.
@@ -1426,6 +1430,10 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
   /// processing State.VF elements.
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of this VPWidenRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
   unsigned getOpcode() const { return Opcode; }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bc57ea4d52471e4..c9d603612aeceaa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1140,6 +1140,80 @@ void VPWidenRecipe::execute(VPTransformState &State) {
 #endif
 }
 
+InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
+                                           VPCostContext &Ctx) const {
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  switch (Opcode) {
+  case Instruction::FNeg: {
+    Type *VectorTy =
+        ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+    return Ctx.TTI.getArithmeticInstrCost(
+        Opcode, VectorTy, CostKind,
+        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None});
+  }
+
+  case Instruction::UDiv:
+  case Instruction::SDiv:
+  case Instruction::SRem:
+  case Instruction::URem:
+    // More complex computation, let the legacy cost-model handle this for now.
+    return Ctx.getLegacyCost(cast<Instruction>(getUnderlyingValue()), VF);
+  case Instruction::Add:
+  case Instruction::FAdd:
+  case Instruction::Sub:
+  case Instruction::FSub:
+  case Instruction::Mul:
+  case Instruction::FMul:
+  case Instruction::FDiv:
+  case Instruction::FRem:
+  case Instruction::Shl:
+  case Instruction::LShr:
+  case Instruction::AShr:
+  case Instruction::And:
+  case Instruction::Or:
+  case Instruction::Xor: {
+    VPValue *RHS = getOperand(1);
+    // Certain instructions can be cheaper to vectorize if they have a constant
+    // second vector operand. One example of this are shifts on x86.
+    TargetTransformInfo::OperandValueInfo RHSInfo = {
+        TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None};
+    if (RHS->isLiveIn())
+      RHSInfo = Ctx.TTI.getOperandInfo(RHS->getLiveInIRValue());
+
+    if (RHSInfo.Kind == TargetTransformInfo::OK_AnyValue &&
+        getOperand(1)->isDefinedOutsideVectorRegions())
+      RHSInfo.Kind = TargetTransformInfo::OK_UniformValue;
+    Type *VectorTy =
+        ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+
+    SmallVector<const Value *, 4> Operands;
+    if (CtxI)
+      Operands.append(CtxI->value_op_begin(), CtxI->value_op_end());
+    return Ctx.TTI.getArithmeticInstrCost(
+        Opcode, VectorTy, CostKind,
+        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+        RHSInfo, Operands, CtxI, &Ctx.TLI);
+  }
+  case Instruction::Freeze: {
+    // This opcode is unknown. Assume that it is the same as 'mul'.
+    Type *VectorTy =
+        ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+    return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
+  }
+  case Instruction::ICmp:
+  case Instruction::FCmp: {
+    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+    Type *VectorTy = ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
+    return Ctx.TTI.getCmpSelInstrCost(Opcode, VectorTy, nullptr, getPredicate(),
+                                      CostKind, CtxI);
+  }
+  default:
+    llvm_unreachable("Unsupported opcode for instruction");
+  }
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
                           VPSlotTracker &SlotTracker) const {
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll b/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
index 497ade4f2f613ca..84150765d77973b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
@@ -131,6 +131,122 @@ define i32 @fneg(i32 %arg) {
   ret i32 undef
 }
 
+define i32 @fmulfneg(i32 %arg) {
+; CHECK-LABEL: 'fmulfneg'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fneg half undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16M = fmul half %F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16M = fmul <2 x half> %V2F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16M = fmul <4 x half> %V4F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16M = fmul <8 x half> %V8F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fneg <16 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16M = fmul <16 x half> %V16F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fneg float undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32M = fmul float %F32, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32M = fmul <2 x float> %V2F32, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32M = fmul <4 x float> %V4F32, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fneg <8 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32M = fmul <8 x float> %V8F32, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fneg double undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64M = fmul double %F64, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64M = fmul <2 x double> %V2F64, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fneg <4 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64M = fmul <4 x double> %V4F64, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F16 = fneg half undef
+  %F16M = fmul half %F16, undef
+  %V2F16 = fneg <2 x half> undef
+  %V2F16M = fmul <2 x half> %V2F16, undef
+  %V4F16 = fneg <4 x half> undef
+  %V4F16M = fmul <4 x half> %V4F16, undef
+  %V8F16 = fneg <8 x half> undef
+  %V8F16M = fmul <8 x half> %V8F16, undef
+  %V16F16 = fneg <16 x half> undef
+  %V16F16M = fmul <16 x half> %V16F16, undef
+
+  %F32 = fneg float undef
+  %F32M = fmul float %F32, undef
+  %V2F32 = fneg <2 x float> undef
+  %V2F32M = fmul <2 x float> %V2F32, undef
+  %V4F32 = fneg <4 x float> undef
+  %V4F32M = fmul <4 x float> %V4F32, undef
+  %V8F32 = fneg <8 x float> undef
+  %V8F32M = fmul <8 x float> %V8F32, undef
+
+  %F64 = fneg double undef
+  %F64M = fmul double %F64, undef
+  %V2F64 = fneg <2 x double> undef
+  %V2F64M = fmul <2 x double> %V2F64, undef
+  %V4F64 = fneg <4 x double> undef
+  %V4F64M = fmul <4 x double> %V4F64, undef
+
+  ret i32 undef
+}
+
+define i32 @fnegfmul(i32 %arg) {
+; CHECK-LABEL: 'fnegfmul'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16M = fmul half undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fneg half %F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16M = fmul <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> %V2F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16M = fmul <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> %V4F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16M = fmul <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> %V8F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16M = fmul <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fneg <16 x half> %V16F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32M = fmul float undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fneg float %F32M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32M = fmul <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> %V2F32M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32M = fmul <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> %V4F32M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32M = fmul <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fneg <8 x float> %V8F32M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64M = fmul double undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fneg double %F64M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64M = fmul <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> %V2F64M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64M = fmul <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fneg <4 x double> %V4F64M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F16M = fmul half undef, undef
+  %F16 = fneg half %F16M
+  %V2F16M = fmul <2 x half> undef, undef
+  %V2F16 = fneg <2 x half> %V2F16M
+  %V4F16M = fmul <4 x half> undef, undef
+  %V4F16 = fneg <4 x half> %V4F16M
+  %V8F16M = fmul <8 x half> undef, undef
+  %V8F16 = fneg <8 x half> %V8F16M
+  %V16F16M = fmul <16 x half> undef, undef
+  %V16F16 = fneg <16 x half> %V16F16M
+
+  %F32M = fmul float undef, undef
+  %F32 = fneg float %F32M
+  %V2F32M = fmul <2 x float> undef, undef
+  %V2F32 = fneg <2 x float> %V2F32M
+  %V4F32M = fmul <4 x float> undef, undef
+  %V4F32 = fneg <4 x float> %V4F32M
+  %V8F32M = fmul <8 x float> undef, undef
+  %V8F32 = fneg <8 x float> %V8F32M
+
+  %F64M = fmul double undef, undef
+  %F64 = fneg double %F64M
+  %V2F64M = fmul <2 x double> undef, undef
+  %V2F64 = fneg <2 x double> %V2F64M
+  %V4F64M = fmul <4 x double> undef, undef
+  %V4F64 = fneg <4 x double> %V4F64M
+
+  ret i32 undef
+}
+
 define i32 @fmul(i32 %arg) {
 ; CHECK-LABEL: 'fmul'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fmul half undef, undef
diff --git a/llvm/test/Analysis/CostModel/X86/icmp-codesize.ll b/llvm/test/Analysis/CostModel/X86/icmp-codesize.ll
index d952e9896d21592..2dc6737a3d8a07d 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp-codesize.ll
@@ -2089,3 +2089,335 @@ define i32 @cmp_int_ult(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64
   ret i32 undef
 }
 
+define i32 @scmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; AVX1-LABEL: 'scmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'scmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'scmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'scmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'scmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'scmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.scmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.scmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.scmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
+define i32 @ucmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; AVX1-LABEL: 'ucmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ucmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ucmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ucmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'ucmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'ucmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.ucmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.ucmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.ucmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
diff --git a/llvm/test/Analysis/CostModel/X86/icmp-latency.ll b/llvm/test/Analysis/CostModel/X86/icmp-latency.ll
index 89cc6e893b31814..726a6dd782a4f7c 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp-latency.ll
@@ -2319,3 +2319,427 @@ define i32 @cmp_int_ult(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64
   ret i32 undef
 }
 
+define i32 @scmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; SSE42-LABEL: 'scmp_int'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'scmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'scmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'scmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'scmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'scmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'scmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'scmp_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 372 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 744 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 360 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.scmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.scmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.scmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
+define i32 @ucmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; SSE42-LABEL: 'ucmp_int'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'ucmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ucmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ucmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ucmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'ucmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'ucmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'ucmp_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 372 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 744 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 360 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.ucmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.ucmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.ucmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
diff --git a/llvm/test/Analysis/CostModel/X86/icmp-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/icmp-sizelatency.ll
index 2cac3af5c01b824..0deaad5991fb2f1 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp-sizelatency.ll
@@ -2089,3 +2089,335 @@ define i32 @cmp_int_ult(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64
   ret i32 undef
 }
 
+define i32 @scmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; AVX1-LABEL: 'scmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'scmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'scmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'scmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'scmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'scmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.scmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.scmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.scmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
+define i32 @ucmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; AVX1-LABEL: 'ucmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ucmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ucmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ucmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'ucmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'ucmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.ucmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.ucmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.ucmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
diff --git a/llvm/test/Analysis/CostModel/X86/icmp.ll b/llvm/test/Analysis/CostModel/X86/icmp.ll
index b1d13ffa605ed08..599895c2b5705a2 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp.ll
@@ -3055,3 +3055,611 @@ define i32 @cmp_int_ult(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64
   ret i32 undef
 }
 
+define i32 @scmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; SSE2-LABEL: 'scmp_int'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'scmp_int'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'scmp_int'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'scmp_int'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'scmp_int'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'scmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'scmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'scmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'scmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'scmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'scmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'scmp_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 372 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 744 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 360 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.scmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.scmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.scmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
+define i32 @ucmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; SSE2-LABEL: 'ucmp_int'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'ucmp_int'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'ucmp_int'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'ucmp_int'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'ucmp_int'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'ucmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ucmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ucmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ucmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'ucmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'ucmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'ucmp_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 372 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 744 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 360 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.ucmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.ucmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.ucmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
diff --git a/llvm/test/Assembler/getInt.ll b/llvm/test/Assembler/getInt.ll
deleted file mode 100644
index 8e2537ae6cf1d8e..000000000000000
--- a/llvm/test/Assembler/getInt.ll
+++ /dev/null
@@ -1,3 +0,0 @@
-; RUN: not opt < %s 2>&1 | grep 'not a number, or does not fit in an unsigned int'
-
-target datalayout = "p:4294967296:64:64"
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ext.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ext.mir
index a431a6764d06f7b..629238126380c77 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ext.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ext.mir
@@ -251,8 +251,9 @@ body:             |
     ; CHECK: liveins: $h0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
-    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s16)
-    ; CHECK-NEXT: $x0 = COPY [[ZEXT]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[ZEXT]](s32)
+    ; CHECK-NEXT: $x0 = COPY [[SEXT]](s64)
     %0:_(s16) = COPY $h0
     %1:_(s32) = G_ZEXT %0(s16)
     %2:_(s64) = G_SEXT %1(s32)
@@ -267,8 +268,10 @@ body:             |
     ; CHECK: liveins: $h0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
-    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s16)
-    ; CHECK-NEXT: $x0 = COPY [[ZEXT]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[ZEXT]], 11
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[ASSERT_ZEXT]](s32)
+    ; CHECK-NEXT: $x0 = COPY [[SEXT]](s64)
     %0:_(s16) = COPY $h0
     %1:_(s32) = G_ZEXT %0(s16)
     %2:_(s32) = G_ASSERT_ZEXT %1(s32), 11
@@ -284,8 +287,9 @@ body:             |
     ; CHECK: liveins: $s0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $s0
-    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(<2 x s64>) = G_ZEXT [[COPY]](<2 x s16>)
-    ; CHECK-NEXT: $q0 = COPY [[ZEXT]](<2 x s64>)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(<2 x s32>) = G_ZEXT [[COPY]](<2 x s16>)
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(<2 x s64>) = G_SEXT [[ZEXT]](<2 x s32>)
+    ; CHECK-NEXT: $q0 = COPY [[SEXT]](<2 x s64>)
     %0:_(<2 x s16>) = COPY $s0
     %1:_(<2 x s32>) = G_ZEXT %0(<2 x s16>)
     %2:_(<2 x s64>) = G_SEXT %1(<2 x s32>)
@@ -340,3 +344,67 @@ body:             |
     %2:_(<2 x s64>) = G_ZEXT %1(<2 x s32>)
     $q0 = COPY %2(<2 x s64>)
 ...
+---
+name:            test_combine_zext_anyext
+body:             |
+  bb.1:
+  liveins: $h0
+    ; CHECK-LABEL: name: test_combine_zext_anyext
+    ; CHECK: liveins: $h0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: $x0 = COPY [[ZEXT]](s64)
+    %0:_(s16) = COPY $h0
+    %1:_(s32) = G_ANYEXT %0(s16)
+    %2:_(s64) = G_ZEXT %1(s32)
+    $x0 = COPY %2(s64)
+...
+---
+name:            test_combine_sext_anyext
+body:             |
+  bb.1:
+  liveins: $h0
+    ; CHECK-LABEL: name: test_combine_sext_anyext
+    ; CHECK: liveins: $h0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[COPY]](s16)
+    ; CHECK-NEXT: $x0 = COPY [[SEXT]](s64)
+    %0:_(s16) = COPY $h0
+    %1:_(s32) = G_ANYEXT %0(s16)
+    %2:_(s64) = G_SEXT %1(s32)
+    $x0 = COPY %2(s64)
+...
+---
+name:            test_combine_nneg_zext_anyext
+body:             |
+  bb.1:
+  liveins: $h0
+    ; CHECK-LABEL: name: test_combine_nneg_zext_anyext
+    ; CHECK: liveins: $h0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: %2:_(s64) = nneg G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: $x0 = COPY %2(s64)
+    %0:_(s16) = COPY $h0
+    %1:_(s32) = G_ANYEXT %0(s16)
+    %2:_(s64) = nneg G_ZEXT %1(s32)
+    $x0 = COPY %2(s64)
+...
+---
+name:            test_combine_anyext_nneg_zext
+body:             |
+  bb.1:
+  liveins: $h0
+    ; CHECK-LABEL: name: test_combine_anyext_nneg_zext
+    ; CHECK: liveins: $h0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: %2:_(s64) = nneg G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: $x0 = COPY %2(s64)
+    %0:_(s16) = COPY $h0
+    %1:_(s32) = nneg G_ZEXT %0(s16)
+    %2:_(s64) = G_ANYEXT %1(s32)
+    $x0 = COPY %2(s64)
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
index 1eb445c03efcd6f..d5356711585fafb 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
@@ -203,8 +203,8 @@ body:             |
     ; CHECK-LABEL: name: test_icmp_and_icmp_9_2
     ; CHECK: liveins: $x0, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: $x0 = COPY [[C]](s64)
+    ; CHECK-NEXT: %zext:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: $x0 = COPY %zext(s64)
     %0:_(s64) = COPY $x0
     %nine:_(s64) = G_CONSTANT i64 9
     %two:_(s64) = G_CONSTANT i64 2
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
index ec66892b98fc7d3..bc4b5ae7c066a6e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
@@ -76,9 +76,9 @@ body:             |
     ; CHECK: liveins: $w0, $w1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w2
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: %o_wide:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: $w0 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: $w1 = COPY [[C]](s32)
+    ; CHECK-NEXT: $w1 = COPY %o_wide(s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %0:_(s32) = COPY $w0
     %1:_(s32) = COPY $w1
@@ -101,10 +101,10 @@ body:             |
     ; CHECK: liveins: $w0, $w1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
-    ; CHECK-NEXT: %const:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: %o_wide:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: $w0 = COPY [[COPY]](s32)
     ; CHECK-NEXT: $w1 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: $w2 = COPY %const(s32)
+    ; CHECK-NEXT: $w2 = COPY %o_wide(s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %0:_(s32) = COPY $w0
     %const:_(s32) = G_CONSTANT i32 0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
index d465e0237201be6..b681e3b22311743 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
@@ -257,10 +257,10 @@ define i32 @udiv_div_by_180(i32 %x)
 ;
 ; GISEL-LABEL: udiv_div_by_180:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    uxtb w8, w0
-; GISEL-NEXT:    mov w9, #5826 // =0x16c2
-; GISEL-NEXT:    movk w9, #364, lsl #16
-; GISEL-NEXT:    umull x8, w8, w9
+; GISEL-NEXT:    mov w8, #5826 // =0x16c2
+; GISEL-NEXT:    and w9, w0, #0xff
+; GISEL-NEXT:    movk w8, #364, lsl #16
+; GISEL-NEXT:    umull x8, w9, w8
 ; GISEL-NEXT:    lsr x0, x8, #32
 ; GISEL-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; GISEL-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/endian_fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/endian_fallback.ll
new file mode 100644
index 000000000000000..6c27b4dd85f9b09
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/endian_fallback.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple aarch64-unknown-linux-musl -O0 -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LE
+; RUN: llc -mtriple aarch64_be-unknown-linux-musl -O0 -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-BE
+
+; Make sure we fall-back to SDAG for BE targets.
+
+; CHECK-LE-NOT: warning: Instruction selection used fallback path for foo
+; CHECK-BE: warning: Instruction selection used fallback path for foo
+
+define <4 x i6> @foo(float %0, <4 x i6> %1) {
+; CHECK-LE-LABEL: foo:
+; CHECK-LE:       // %bb.0:
+; CHECK-LE-NEXT:    fmov d0, d1
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: foo:
+; CHECK-BE:       // %bb.0:
+; CHECK-BE-NEXT:    fmov d0, d1
+; CHECK-BE-NEXT:    ret
+  ret <4 x i6> %1
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir b/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir
index 1e9ef108c99cea6..f0fbfecf219fd94 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir
@@ -151,9 +151,9 @@ body:             |
     ; CHECK-LABEL: name: ptr_add_chain
     ; CHECK: liveins: $x0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 1
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: %dont_fold_me:_(p0) = G_PTR_ADD [[GV]], [[C]](s64)
+    ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 4
+    ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: %dont_fold_me:_(p0) = G_PTR_ADD [[GV]], %offset(s64)
     ; CHECK-NEXT: $x0 = COPY %dont_fold_me(p0)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %global:_(p0) = G_GLOBAL_VALUE @g
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-sameopcode-hands-crash.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-sameopcode-hands-crash.mir
index f8510a2bd3e7137..2d286ede38d5dea 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-sameopcode-hands-crash.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-sameopcode-hands-crash.mir
@@ -9,6 +9,8 @@ body:             |
   bb.1:
     ; CHECK-LABEL: name: crash_fn
     ; CHECK: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
     ; CHECK-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[C]](p0) :: (load (s16), align 8)
     ; CHECK-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[C]](p0) :: (load (s16))
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ZEXTLOAD1]], [[ZEXTLOAD]]
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir
index ce7450c69050394..b8020bd50a2b0d6 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir
@@ -64,21 +64,28 @@ body:             |
     ; LOWER-NEXT: {{  $}}
     ; LOWER-NEXT: %reg0:_(s64) = COPY $x0
     ; LOWER-NEXT: %cmp_lhs:_(s64) = G_SEXT_INREG %reg0, 8
+    ; LOWER-NEXT: %reg1:_(s64) = COPY $x1
+    ; LOWER-NEXT: %cmp1:_(s32) = G_ICMP intpred(sge), %cmp_lhs(s64), %reg1
     ; LOWER-NEXT: %add:_(s64) = G_ADD %cmp_lhs, %reg0
     ; LOWER-NEXT: %cmp2:_(s32) = G_ICMP intpred(sge), %cmp_lhs(s64), %add
     ; LOWER-NEXT: $w0 = COPY %cmp2(s32)
-    ; LOWER-NEXT: RET_ReallyLR implicit $w0
+    ; LOWER-NEXT: $w1 = COPY %cmp1(s32)
+    ; LOWER-NEXT: RET_ReallyLR implicit $w0, implicit $w1
     ;
     ; SELECT-LABEL: name: dont_swap_more_than_one_use
     ; SELECT: liveins: $x0, $x1
     ; SELECT-NEXT: {{  $}}
     ; SELECT-NEXT: %reg0:gpr64 = COPY $x0
     ; SELECT-NEXT: %cmp_lhs:gpr64 = SBFMXri %reg0, 0, 7
+    ; SELECT-NEXT: %reg1:gpr64 = COPY $x1
+    ; SELECT-NEXT: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr %cmp_lhs, %reg1, implicit-def $nzcv
+    ; SELECT-NEXT: %cmp1:gpr32 = CSINCWr $wzr, $wzr, 11, implicit $nzcv
     ; SELECT-NEXT: %add:gpr64 = ADDXrr %cmp_lhs, %reg0
-    ; SELECT-NEXT: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr %cmp_lhs, %add, implicit-def $nzcv
+    ; SELECT-NEXT: [[SUBSXrr1:%[0-9]+]]:gpr64 = SUBSXrr %cmp_lhs, %add, implicit-def $nzcv
     ; SELECT-NEXT: %cmp2:gpr32 = CSINCWr $wzr, $wzr, 11, implicit $nzcv
     ; SELECT-NEXT: $w0 = COPY %cmp2
-    ; SELECT-NEXT: RET_ReallyLR implicit $w0
+    ; SELECT-NEXT: $w1 = COPY %cmp1
+    ; SELECT-NEXT: RET_ReallyLR implicit $w0, implicit $w1
     %reg0:_(s64) = COPY $x0
     %cmp_lhs:_(s64) = G_SEXT_INREG %reg0, 8
     %reg1:_(s64) = COPY $x1
@@ -88,7 +95,8 @@ body:             |
     %cmp2:_(s32) = G_ICMP intpred(sge), %cmp_lhs(s64), %add
 
     $w0 = COPY %cmp2(s32)
-    RET_ReallyLR implicit $w0
+    $w1 = COPY %cmp1(s32)
+    RET_ReallyLR implicit $w0, implicit $w1
 
 ...
 ---
diff --git a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
index d39029163a47aa6..2181eaaee7db686 100644
--- a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
@@ -214,9 +214,8 @@ define void @t17(i64 %a) {
 define i8 @LdOffset_i8(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrb w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrb w0, [x8, #3704]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
@@ -227,9 +226,8 @@ define i8 @LdOffset_i8(ptr %a)  {
 define i32 @LdOffset_i8_zext32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_zext32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrb w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrb w0, [x8, #3704]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
@@ -255,9 +253,8 @@ define i32 @LdOffset_i8_sext32(ptr %a)  {
 define i64 @LdOffset_i8_zext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_zext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrb w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrb w0, [x8, #3704]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
diff --git a/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir b/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir
index 488f1ffdb52f3b2..473c74323d93998 100644
--- a/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir
+++ b/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir
@@ -14,9 +14,8 @@ body:             |
     ; CHECK-LABEL: name: LdOffset
     ; CHECK: liveins: $x0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: renamable $w8 = MOVZWi 56952, 0
-    ; CHECK-NEXT: renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8
-    ; CHECK-NEXT: renamable $w0 = LDRBBroX killed renamable $x0, killed renamable $x8, 0, 0
+    ; CHECK-NEXT: $x8 = ADDXri $x0, 253, 12
+    ; CHECK-NEXT: renamable $w0 = LDRBBui killed renamable $x8, 3704
     ; CHECK-NEXT: RET undef $lr, implicit $w0
     renamable $w8 = MOVZWi 56952, 0
     renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8
@@ -46,3 +45,23 @@ body:             |
     renamable $w0 = LDRBBroX killed renamable $x0, renamable $x8, 0, 0
     RET undef $lr, implicit $w0
 ...
+
+# Negative test: No MOVZWi used for the const offset
+---
+name:            LdOffset_missing_MOVZ
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0', virtual-reg: '' }
+body:             |
+  bb.0.entry:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: LdOffset_missing_MOVZ
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8
+    ; CHECK-NEXT: renamable $w0 = LDRBBroX killed renamable $x0, killed renamable $x8, 0, 0
+    ; CHECK-NEXT: RET undef $lr, implicit $w0
+    renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8
+    renamable $w0 = LDRBBroX killed renamable $x0, killed renamable $x8, 0, 0
+    RET undef $lr, implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-unsafe-stack-call.mir b/llvm/test/CodeGen/AArch64/machine-outliner-unsafe-stack-call.mir
index 0853b23e52916d5..679c9687e66f039 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-unsafe-stack-call.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-unsafe-stack-call.mir
@@ -25,10 +25,10 @@ body:             |
     ; CHECK-DAG: baz
     $lr = ORRXri $xzr, 1
     BL @foo, implicit-def dead $lr, implicit $sp
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
   bb.1:
     BL @bar, implicit-def dead $lr, implicit $sp
     $x11 = ADDXri $sp, 48, 0;
@@ -55,10 +55,10 @@ body:             |
     ; CHECK-DAG: baz
     $lr = ORRXri $xzr, 1
     BL @foo, implicit-def dead $lr, implicit $sp
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
   bb.1:
     BL @bar, implicit-def dead $lr, implicit $sp
     $x11 = ADDXri $sp, 48, 0;
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-elf-pauthabi.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-elf-pauthabi.ll
index b2ebf1fc041137b..e5f675e62577d0c 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-elf-pauthabi.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-elf-pauthabi.ll
@@ -10,7 +10,7 @@
 !llvm.module.flags = !{!0, !1}
 
 !0 = !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
-!1 = !{i32 1, !"aarch64-elf-pauthabi-version", i32 341}
+!1 = !{i32 1, !"aarch64-elf-pauthabi-version", i32 1365}
 
 ; ASM: .section .note.gnu.property,"a",@note
 ; ASM-NEXT: .p2align 3, 0x0
@@ -22,12 +22,12 @@
 ; ASM-NEXT: .word 3221225473
 ; ASM-NEXT: .word 16
 ; ASM-NEXT: .xword 268435458
-; ASM-NEXT: .xword 341
+; ASM-NEXT: .xword 1365
 
 ; OBJ: Displaying notes found in: .note.gnu.property
 ; OBJ-NEXT:   Owner                 Data size	Description
 ; OBJ-NEXT:   GNU                   0x00000018	NT_GNU_PROPERTY_TYPE_0 (property note)
-; OBJ-NEXT:   AArch64 PAuth ABI core info: platform 0x10000002 (llvm_linux), version 0x155 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)
+; OBJ-NEXT:   AArch64 PAuth ABI core info: platform 0x10000002 (llvm_linux), version 0x555 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, !PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)
 
 ; ERR: either both or no 'aarch64-elf-pauthabi-platform' and 'aarch64-elf-pauthabi-version' module flags must be present
 
diff --git a/llvm/test/CodeGen/AArch64/preserve.ll b/llvm/test/CodeGen/AArch64/preserve.ll
index a8acdc1df976068..5839bb5440a9b05 100644
--- a/llvm/test/CodeGen/AArch64/preserve.ll
+++ b/llvm/test/CodeGen/AArch64/preserve.ll
@@ -4,19 +4,19 @@
 target triple = "aarch64-unknown-unknown"
 declare void @bar1()
 define preserve_mostcc void @baz() #0 {
-; CHECK: baz Clobbered Registers: $ffr $fpcr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+; CHECK: baz Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
   call void @bar1()
   call void @bar2()
   ret void
 }
 define preserve_allcc void @foo() #0 {
-; CHECK: foo Clobbered Registers: $ffr $fpcr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+; CHECK: foo Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
   call void @bar1()
   call void @bar2()
   ret void
 }
 define preserve_nonecc void @qux() #0 {
-;  CHECK: qux Clobbered Registers: $ffr $fpcr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10 $w11 $w12 $w13 $w14 $w15 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x9 $x10 $x11 $x12 $x13 $x14 $x15 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+;  CHECK: qux Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10 $w11 $w12 $w13 $w14 $w15 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x9 $x10 $x11 $x12 $x13 $x14 $x15 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
 
     call void @bar1()
     call void @bar2()
diff --git a/llvm/test/CodeGen/AArch64/print-mrs-system-register.ll b/llvm/test/CodeGen/AArch64/print-mrs-system-register.ll
deleted file mode 100644
index 3411ed6161bdbdf..000000000000000
--- a/llvm/test/CodeGen/AArch64/print-mrs-system-register.ll
+++ /dev/null
@@ -1,11 +0,0 @@
-; RUN: llc -mtriple=arm64-apple-darwin %s -o - | FileCheck %s
-
-; CHECK: mrs x0, CPM_IOACC_CTL_EL3
-
-define void @foo1() #0 {
-entry:
-  tail call void asm sideeffect "mrs x0, cpm_ioacc_ctl_el3", ""()
-  ret void
-}
-
-attributes #0 = { "target-cpu"="cyclone" }
diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
index 8be63b04d8ce588..4dd2a896cd81cbc 100644
--- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
+++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
@@ -33,8 +33,7 @@ define noundef i1 @logger(i32 noundef %logLevel, ptr %ea, ptr %pll) {
 ; CHECK-GI-NEXT:    b.hi .LBB1_2
 ; CHECK-GI-NEXT:  // %bb.1: // %land.rhs
 ; CHECK-GI-NEXT:    ldr x8, [x1]
-; CHECK-GI-NEXT:    ldrb w8, [x8]
-; CHECK-GI-NEXT:    and w0, w8, #0x1
+; CHECK-GI-NEXT:    ldrb w0, [x8]
 ; CHECK-GI-NEXT:  .LBB1_2: // %land.end
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll b/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll
new file mode 100644
index 000000000000000..c32e9cbc053939c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll
@@ -0,0 +1,55 @@
+; RUN: llc -mtriple=aarch64-darwin -mattr=+sve -mattr=+sme -verify-machineinstrs < %s | FileCheck %s
+
+declare void @normal_callee();
+
+define void @locally_streaming_fn() #0 {
+; CHECK-LABEL: locally_streaming_fn:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    stp d15, d14, [sp, #-96]! ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d13, d12, [sp, #16] ; 16-byte Folded Spill
+; CHECK-NEXT:    lsr x9, x9, #3
+; CHECK-NEXT:    stp d11, d10, [sp, #32] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #48] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x9, [sp, #64] ; 16-byte Folded Spill
+; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    str x9, [sp, #80] ; 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_offset b8, -40
+; CHECK-NEXT:    .cfi_offset b9, -48
+; CHECK-NEXT:    .cfi_offset b10, -56
+; CHECK-NEXT:    .cfi_offset b11, -64
+; CHECK-NEXT:    .cfi_offset b12, -72
+; CHECK-NEXT:    .cfi_offset b13, -80
+; CHECK-NEXT:    .cfi_offset b14, -88
+; CHECK-NEXT:    .cfi_offset b15, -96
+; CHECK-NEXT:    smstart sm
+; CHECK-NEXT:    .cfi_offset vg, -24
+; CHECK-NEXT:    smstop sm
+; CHECK-NEXT:    bl _normal_callee
+; CHECK-NEXT:    smstart sm
+; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    smstop sm
+; CHECK-NEXT:    ldp d9, d8, [sp, #48] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #64] ; 8-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #32] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #16] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #96 ; 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
+; CHECK-NEXT:    ret
+  call void @normal_callee()
+  ret void
+}
+
+attributes #0 = { "aarch64_pstate_sm_body" uwtable(async) }
diff --git a/llvm/test/CodeGen/AArch64/verify-memop.mir b/llvm/test/CodeGen/AArch64/verify-memop.mir
new file mode 100644
index 000000000000000..90aa5e64a7f7209
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/verify-memop.mir
@@ -0,0 +1,51 @@
+# RUN: not --crash llc -mtriple=aarch64 -run-pass machineverifier -o /dev/null %s 2>&1 | FileCheck -implicit-check-not="Bad machine code"  %s
+
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRSui $s1, $x0, 4096
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRSui $s1, $x0, -1
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0, $w1 = LDRWpre $x0(tied-def 0), 256
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0, $w1 = LDRWpre $x0(tied-def 0), -257
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0, $w1 = LDRWpre $x0(tied-def 0), 256
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0, $w1 = LDRWpre $x0(tied-def 0), -257
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRBBui $w1, $x0, 4096
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRBBui $w1, $x0, -1
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRHHui $w1, $x0, 4096
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRHHui $w1, $x0, -1
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0 = STRSpost $s1, $x0(tied-def 0), 256
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0 = STRSpost $s1, $x0(tied-def 0), -257
+
+---
+name:            testoffsets
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $d0, $s1, $q2, $x0, $w1
+
+    STRSui $s1, $x0, 4095
+    STRSui $s1, $x0, 4096
+    STRSui $s1, $x0, 0
+    STRSui $s1, $x0, -1
+    $x0, $w1 = LDRWpre $x0, 256
+    $x0, $w1 = LDRWpre $x0, -257
+    $x0, $w1 = LDRWpre $x0, 256
+    $x0, $w1 = LDRWpre $x0, -257
+    STRBBui $w1, $x0, 4096
+    STRBBui $w1, $x0, -1
+    STRHHui $w1, $x0, 4096
+    STRHHui $w1, $x0, -1
+    $x0 = STRSpost $s1, $x0, 256
+    $x0 = STRSpost $s1, $x0, -257
+    RET undef $lr, implicit $x0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir
index 24c4d668885f439..0b3b4288556bfef 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir
@@ -509,16 +509,14 @@ body: |
     ; SIVI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load (s32), addrspace 4)
     ; SIVI-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[FRAME_INDEX]](p5)
     ; SIVI-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[LOAD]](s32)
-    ; SIVI-NEXT: [[COPY2:%[0-9]+]]:_(p0) = COPY [[MV]](p0)
-    ; SIVI-NEXT: $vgpr0_vgpr1 = COPY [[COPY2]](p0)
+    ; SIVI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p0)
     ; GFX9-LABEL: name: test_addrspacecast_p5_fi_to_p0
     ; GFX9: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0
     ; GFX9-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64(s64) = S_MOV_B64 $src_private_base
     ; GFX9-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[S_MOV_B64_]](s64)
     ; GFX9-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[FRAME_INDEX]](p5)
     ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[UV1]](s32)
-    ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY [[MV]](p0)
-    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[COPY]](p0)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p0)
     %0:_(p5) = G_FRAME_INDEX %stack.0
     %1:_(p0) = G_ADDRSPACE_CAST %0
     $vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
index f312f3138c08d34..9fa3eb22a554a80 100644
--- a/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
@@ -1,8 +1,9 @@
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=FAIL
-; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=FAIL
+; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1030 -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefixes=SDAG-FAIL
+; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=GISEL-FAIL
 
-define amdgpu_gs void @test_fptrunc_round_legalization(double %a, i32 %data0, <4 x i32> %data1, ptr addrspace(1) %out) {
-; FAIL: LLVM ERROR: Cannot select
+define amdgpu_gs void @test_fptrunc_round_f64(double %a, ptr addrspace(1) %out) {
+; SDAG-FAIL: LLVM ERROR: Cannot select
+; GISEL-FAIL: unable to legalize instruction
   %res = call half @llvm.fptrunc.round.f64(double %a, metadata !"round.upward")
   store half %res, ptr addrspace(1) %out, align 4
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
index 4ac9dc8565ff40e..fd1b99a9f30eb7c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
@@ -1,10 +1,9 @@
-; RUN: llc -mtriple=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=verde < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=verde < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}mbcnt_intrinsics:
-; GCN: v_mbcnt_lo_u32_b32{{(_e64)*}} [[LO:v[0-9]+]], -1, 0
-; SI: v_mbcnt_hi_u32_b32_e32 {{v[0-9]+}}, -1, [[LO]]
-; VI: v_mbcnt_hi_u32_b32 {{v[0-9]+}}, -1, [[LO]]
+; GCN: v_mbcnt_lo_u32_b32_e64 [[LO:v[0-9]+]], -1, 0
+; GCN: v_mbcnt_hi_u32_b32_e32 {{v[0-9]+}}, -1, [[LO]]
 define amdgpu_ps void @mbcnt_intrinsics(ptr addrspace(4) inreg %arg, ptr addrspace(4) inreg %arg1, ptr addrspace(4) inreg %arg2, i32 inreg %arg3) {
 main_body:
   %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
@@ -80,13 +79,25 @@ define i32 @mbcnt_hi_known_bits_3(i32 %x) #0 {
 
 ; GCN-LABEL: {{^}}mbcnt_hi_known_bits_4:
 ; GCN: v_mbcnt_hi_u32_b32
-; GCN: v_and_b32_e32
+; GCN-NOT: v_and_b32_e32
 define i32 @mbcnt_hi_known_bits_4(i32 %x) #0 {
   %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 %x, i32 15)
   %mask = and i32 %hi, 63
   ret i32 %mask
 }
 
+; TODO: Special case mbcnt.lo feeding into mbcnt.hi to remove this AND.
+; GCN-LABEL: {{^}}mbcnt_lo_hi_known_bits_1:
+; GCN: v_mbcnt_lo_u32_b32
+; GCN: v_mbcnt_hi_u32_b32
+; GCN: v_and_b32_e32
+define i32 @mbcnt_lo_hi_known_bits_1(i32 %x) #0 {
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 %x, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 %x, i32 %lo)
+  %mask = and i32 %hi, 63
+  ret i32 %mask
+}
+
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0
 declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #0
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index b8c16d2ed3b2f1f..71d0ee524bab73b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -3,6 +3,15 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=CHECK,SDAG %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
 
+define amdgpu_gs half @v_fptrunc_round_f32_to_f16_tonearest(float %a) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_tonearest:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    ; return to shader part epilog
+  %res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.tonearest")
+  ret half %res
+}
+
 define amdgpu_gs half @v_fptrunc_round_f32_to_f16_upward(float %a) {
 ; CHECK-LABEL: v_fptrunc_round_f32_to_f16_upward:
 ; CHECK:       ; %bb.0:
@@ -23,6 +32,16 @@ define amdgpu_gs half @v_fptrunc_round_f32_to_f16_downward(float %a) {
   ret half %res
 }
 
+define amdgpu_gs half @v_fptrunc_round_f32_to_f16_towardzero(float %a) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_towardzero:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    ; return to shader part epilog
+  %res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.towardzero")
+  ret half %res
+}
+
 define amdgpu_gs void @v_fptrunc_round_f32_to_f16_upward_multiple_calls(float %a, float %b, ptr addrspace(1) %out) {
 ; CHECK-LABEL: v_fptrunc_round_f32_to_f16_upward_multiple_calls:
 ; CHECK:       ; %bb.0:
@@ -45,6 +64,50 @@ define amdgpu_gs void @v_fptrunc_round_f32_to_f16_upward_multiple_calls(float %a
   ret void
 }
 
+define amdgpu_gs void @v_fptrunc_round_f32_to_f16_downward_multiple_calls(float %a, float %b, ptr addrspace(1) %out) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_downward_multiple_calls:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v4, v0
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 2
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
+; CHECK-NEXT:    v_add_f16_e32 v0, v4, v0
+; CHECK-NEXT:    v_add_f16_e32 v0, v1, v0
+; CHECK-NEXT:    global_store_short v[2:3], v0, off
+; CHECK-NEXT:    s_endpgm
+  %res1 = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.upward")
+  %res2 = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.downward")
+  %res3 = call half @llvm.fptrunc.round.f16.f32(float %b, metadata !"round.downward")
+  %res4 = fadd half %res1, %res2
+  %res5 = fadd half %res3, %res4
+  store half %res5, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_gs void @v_fptrunc_round_f32_to_f16_towardzero_multiple_calls(float %a, float %b, ptr addrspace(1) %out) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_towardzero_multiple_calls:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v4, v1
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 1
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
+; CHECK-NEXT:    v_add_f16_e32 v0, v0, v4
+; CHECK-NEXT:    v_add_f16_e32 v0, v1, v0
+; CHECK-NEXT:    global_store_short v[2:3], v0, off
+; CHECK-NEXT:    s_endpgm
+  %res1 = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.towardzero")
+  %res2 = call half @llvm.fptrunc.round.f16.f32(float %b, metadata !"round.towardzero")
+  %res3 = call half @llvm.fptrunc.round.f16.f32(float %b, metadata !"round.upward")
+  %res4 = fadd half %res1, %res2
+  %res5 = fadd half %res3, %res4
+  store half %res5, ptr addrspace(1) %out, align 4
+  ret void
+}
+
 define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addrspace(1) %out) {
 ; CHECK-LABEL: s_fptrunc_round_f32_to_f16_upward:
 ; CHECK:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
index fda33f52bf06d2b..3706c535224438e 100644
--- a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
@@ -3,6 +3,29 @@
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=CHECK
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=GFX11
 
+---
+name: ftrunc_tonearest
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: ftrunc_tonearest
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_ENDPGM 0
+    ;
+    ; GFX11-LABEL: name: ftrunc_tonearest
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 0, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
 ---
 name: ftrunc_upward
 
@@ -25,7 +48,7 @@ body: |
     ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
     ; GFX11-NEXT: S_ENDPGM 0
     $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
-    $vgpr1 = FPTRUNC_UPWARD_PSEUDO $vgpr0, implicit $mode, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 1, implicit $mode, implicit $exec
     S_ENDPGM 0
 ...
 ---
@@ -50,6 +73,31 @@ body: |
     ; GFX11-NEXT: $vgpr0 = V_CVT_F16_F32_t16_e64 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
     ; GFX11-NEXT: S_ENDPGM 0
     $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
-    $vgpr0 = FPTRUNC_DOWNWARD_PSEUDO $vgpr1, implicit $mode, implicit $exec
+    $vgpr0 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr1, 2, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
+---
+name: ftrunc_towardzero
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: ftrunc_towardzero
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; CHECK-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
+    ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_ENDPGM 0
+    ;
+    ; GFX11-LABEL: name: ftrunc_towardzero
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
+    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 3, implicit $mode, implicit $exec
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
index a547c258e3921d7..b7113a65607fc56 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
@@ -1,721 +1,157 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NO-PRELOAD %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-1 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-2 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=4 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-4 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=8 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-8 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940 %s
 
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-NO-PRELOAD %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-1 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-2 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=4 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-4 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=8 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-8 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a %s
 
-define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i8:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i8:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i8:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i8:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i8:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i8:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i8:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i8:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i8:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i8:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) inreg %out, i8 inreg %arg0) #0 {
+; GFX940-LABEL: ptr1_i8:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_and_b32 s0, s4, 0xff
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i8:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_and_b32 s0, s8, 0xff
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %arg0) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i8_zext_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i8_zext_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i8_zext_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i8_zext_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i8_zext_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i8_zext_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i8_zext_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i8_zext_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i8_zext_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i8_zext_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) inreg %out, i8 zeroext inreg %arg0) #0 {
+; GFX940-LABEL: ptr1_i8_zext_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_mov_b32 s0, 0xffff
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i8_zext_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_mov_b32 s0, 0xffff
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i16_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i16_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i16_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i16_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i16_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i16_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i16_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i16_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i16_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i16_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %arg0) #0 {
+; GFX940-LABEL: ptr1_i16_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_and_b32 s0, s4, 0xffff
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i16_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_and_b32 s0, s8, 0xffff
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   %ext = zext i16 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i32_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i32_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i32_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i32_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i32_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i32_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) inreg %out, i32 inreg %arg0) #0 {
+; GFX940-LABEL: ptr1_i32_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i32_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store i32 %arg0, ptr addrspace(1) %out
   ret void
 }
 
 
-define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) %out, i32 %arg1) #0 {
-; GFX940-NO-PRELOAD-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 inreg %arg0, ptr addrspace(1) inreg %out, i32 inreg %arg1) #0 {
+; GFX940-LABEL: i32_ptr1_i32_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_add_i32 s0, s2, s6
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i32_ptr1_i32_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_add_i32 s0, s6, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
+; GFX90a-NEXT:    s_endpgm
   %add = add i32 %arg0, %arg1
   store i32 %add, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0, i16 %arg1) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-NO-PRELOAD-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-NO-PRELOAD-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-PRELOAD-1-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-PRELOAD-1-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-PRELOAD-2-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-PRELOAD-2-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-PRELOAD-4-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-PRELOAD-4-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-PRELOAD-8-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-PRELOAD-8-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-NO-PRELOAD-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-PRELOAD-1-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-1-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-PRELOAD-2-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-2-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-PRELOAD-4-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-4-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-PRELOAD-8-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-8-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %arg0, i16 inreg %arg1) #0 {
+; GFX940-LABEL: ptr1_i16_i16_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    s_and_b32 s1, s4, 0xffff
+; GFX940-NEXT:    s_add_i32 s0, s1, s0
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i16_i16_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    s_and_b32 s1, s8, 0xffff
+; GFX90a-NEXT:    s_add_i32 s0, s1, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   %ext = zext i16 %arg0 to i32
   %ext1 = zext i16 %arg1 to i32
   %add = add i32 %ext, %ext1
@@ -723,1452 +159,878 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 %
   ret void
 }
 
-define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-NO-PRELOAD-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-1-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-2-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-4-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-8-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) inreg %out, <2 x i8> inreg %in) #0 {
+; GFX940-LABEL: ptr1_v2i8_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0
+; GFX940-NEXT:    global_store_short v1, v0, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_v2i8_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 8
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_mov_b32_e32 v1, 0
+; GFX90a-NEXT:    global_store_short v1, v0, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <2 x i8> %in, ptr addrspace(1) %out
   ret void
 }
 
 
-define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspace(4) byref(i32) align(256) %in.byref, i32 %after.offset) #0 {
-; GFX940-NO-PRELOAD-LABEL: byref_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: byref_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: byref_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: byref_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: byref_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: byref_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: byref_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: byref_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: byref_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: byref_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) inreg %out, ptr addrspace(4) byref(i32) align(256) %in.byref, i32 %after.offset) #0 {
+; GFX940-LABEL: byref_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x100
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    v_mov_b32_e32 v2, s5
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dword v0, v2, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: byref_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s1
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_waitcnt vmcnt(0)
+; GFX90a-NEXT:    global_store_dword v0, v2, s[6:7]
+; GFX90a-NEXT:    s_waitcnt vmcnt(0)
+; GFX90a-NEXT:    s_endpgm
   %in = load i32, ptr addrspace(4) %in.byref
   store volatile i32 %in, ptr addrspace(1) %out, align 4
   store volatile i32 %after.offset, ptr addrspace(1) %out, align 4
   ret void
 }
 
+; The second argument is not expected to be preloaded with the current behavior.
 
-define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v8i32_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_nop 1
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v8i32_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_nop 1
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v8i32_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_nop 1
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v8i32_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_nop 1
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v8i32_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_nop 1
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v8i32_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-NO-PRELOAD-NEXT:    s_nop 0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v8i32_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-PRELOAD-1-NEXT:    s_nop 0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v8i32_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-PRELOAD-2-NEXT:    s_nop 0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v8i32_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-PRELOAD-4-NEXT:    s_nop 0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v8i32_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-PRELOAD-8-NEXT:    s_nop 0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @byref_staggered_preload_arg(ptr addrspace(1) inreg %out, ptr addrspace(4) byref(i32) align(256) %in.byref, i32 inreg %after.offset) #0 {
+; GFX940-LABEL: byref_staggered_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x100
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    v_mov_b32_e32 v2, s5
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dword v0, v2, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: byref_staggered_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s1
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_waitcnt vmcnt(0)
+; GFX90a-NEXT:    global_store_dword v0, v2, s[6:7]
+; GFX90a-NEXT:    s_waitcnt vmcnt(0)
+; GFX90a-NEXT:    s_endpgm
+  %in = load i32, ptr addrspace(4) %in.byref
+  store volatile i32 %in, ptr addrspace(1) %out, align 4
+  store volatile i32 %after.offset, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+
+define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture inreg %out, <8 x i32> inreg %in) #0 {
+; GFX940-LABEL: v8i32_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s8
+; GFX940-NEXT:    v_mov_b32_e32 v1, s9
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    v_mov_b32_e32 v3, s11
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_mov_b32_e32 v0, s4
+; GFX940-NEXT:    v_mov_b32_e32 v1, s5
+; GFX940-NEXT:    v_mov_b32_e32 v2, s6
+; GFX940-NEXT:    v_mov_b32_e32 v3, s7
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v8i32_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; GFX90a-NEXT:    s_nop 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s11
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <8 x i32> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 x i16> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v3i16_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-NO-PRELOAD-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v3i16_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-1-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v3i16_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-2-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v3i16_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-4-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v3i16_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-8-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v3i16_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v3i16_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v3i16_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v3i16_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v3i16_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture inreg %out, <3 x i16> inreg %in) #0 {
+; GFX940-LABEL: v3i16_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s5
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] offset:4 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v3i16_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <3 x i16> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 x i32> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v3i32_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v3i32_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v3i32_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v3i32_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v3i32_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v3i32_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v3i32_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v3i32_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v3i32_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v3i32_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture inreg %out, <3 x i32> inreg %in) #0 {
+; GFX940-LABEL: v3i32_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v3i32_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <3 x i32> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 x float> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v3f32_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v3f32_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v3f32_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v3f32_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v3f32_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v3f32_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v3f32_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v3f32_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v3f32_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v3f32_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture inreg %out, <3 x float> inreg %in) #0 {
+; GFX940-LABEL: v3f32_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v3f32_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <3 x float> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 x i8> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v5i8_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-NO-PRELOAD-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v5i8_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-1-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v5i8_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-2-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v5i8_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-4-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v5i8_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-8-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v5i8_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v5i8_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v5i8_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v5i8_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v5i8_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture inreg %out, <5 x i8> inreg %in) #0 {
+; GFX940-LABEL: v5i8_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_mov_b32_e32 v2, s5
+; GFX940-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0
+; GFX940-NEXT:    global_store_byte v1, v2, s[2:3] offset:4 sc0 sc1
+; GFX940-NEXT:    global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v5i8_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 8
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_mov_b32_e32 v1, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s9
+; GFX90a-NEXT:    global_store_byte v1, v2, s[6:7] offset:4
+; GFX90a-NEXT:    global_store_dword v1, v0, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <5 x i8> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x double> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v5f64_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_nop 1
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v5f64_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_nop 1
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v5f64_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_nop 1
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v5f64_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_nop 1
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v5f64_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_nop 1
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v5f64_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-NO-PRELOAD-NEXT:    s_nop 0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v5f64_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-PRELOAD-1-NEXT:    s_nop 0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v5f64_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-PRELOAD-2-NEXT:    s_nop 0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v5f64_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-PRELOAD-4-NEXT:    s_nop 0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v5f64_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-PRELOAD-8-NEXT:    s_nop 0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture inreg %out, <5 x double> inreg %in) #0 {
+; GFX940-LABEL: v5f64_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x60
+; GFX940-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[12:13]
+; GFX940-NEXT:    v_mov_b32_e32 v0, s8
+; GFX940-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v1, s9
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    v_mov_b32_e32 v3, s11
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_mov_b32_e32 v0, s4
+; GFX940-NEXT:    v_mov_b32_e32 v1, s5
+; GFX940-NEXT:    v_mov_b32_e32 v2, s6
+; GFX940-NEXT:    v_mov_b32_e32 v3, s7
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v5f64_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
+; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    global_store_dwordx2 v4, v[2:3], s[6:7] offset:32
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; GFX90a-NEXT:    s_nop 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s11
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <5 x double> %in, ptr addrspace(1) %out, align 8
   ret void
 }
 
-define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v8i8_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v8i8_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v8i8_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v8i8_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v8i8_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v8i8_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v8i8_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v8i8_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v8i8_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v8i8_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) inreg %out, <8 x i8> inreg %in) #0 {
+; GFX940-LABEL: v8i8_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s5, 8
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s5, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s5, 16
+; GFX940-NEXT:    v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX940-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v2, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v8i8_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s9, 8
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s9, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s9, 16
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 8
+; GFX90a-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v2, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <8 x i8> %in, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) #0 {
-; GFX940-NO-PRELOAD-LABEL: i64_kernel_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: i64_kernel_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: i64_kernel_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: i64_kernel_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: i64_kernel_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: i64_kernel_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: i64_kernel_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: i64_kernel_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: i64_kernel_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: i64_kernel_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) inreg %out, i64 inreg %a) #0 {
+; GFX940-LABEL: i64_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i64_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store i64 %a, ptr addrspace(1) %out, align 8
   ret void
 }
 
-define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: f64_kernel_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: f64_kernel_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: f64_kernel_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: f64_kernel_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: f64_kernel_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: f64_kernel_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: f64_kernel_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: f64_kernel_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: f64_kernel_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: f64_kernel_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) inreg %out, double inreg %in) #0 {
+; GFX940-LABEL: f64_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: f64_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store double %in, ptr addrspace(1) %out
   ret void
 }
 
-attributes #0 = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
+define amdgpu_kernel void @half_kernel_preload_arg(ptr addrspace(1) inreg %out, half inreg %in) #0 {
+; GFX940-LABEL: half_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: half_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store half %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, bfloat inreg %in) #0 {
+; GFX940-LABEL: bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store bfloat %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v2bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, <2 x bfloat> inreg %in) #0 {
+; GFX940-LABEL: v2bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v2bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <2 x bfloat> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v3bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, <3 x bfloat> inreg %in) #0 {
+; GFX940-LABEL: v3bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s5
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] offset:4 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v3bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <3 x bfloat> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v6bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, <6 x bfloat> inreg %in) #0 {
+; GFX940-LABEL: v6bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v6bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <6 x bfloat> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @half_v7bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, half inreg %in, <7 x bfloat> inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: half_v7bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s4
+; GFX940-NEXT:    global_store_short v3, v0, s[2:3] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v0, s9
+; GFX940-NEXT:    global_store_short v3, v0, s[10:11] offset:12 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[10:11] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: half_v7bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x20
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
+; GFX90a-NEXT:    global_store_short v3, v0, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s13
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    global_store_short v3, v0, s[0:1] offset:12
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[0:1]
+; GFX90a-NEXT:    s_endpgm
+  store half %in, ptr addrspace(1) %out
+  store <7 x bfloat> %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+define amdgpu_kernel void @i1_kernel_preload_arg(ptr addrspace(1) inreg %out, i1 inreg %in) #0 {
+; GFX940-LABEL: i1_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_and_b32 s0, s4, 1
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_byte v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i1_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_and_b32 s0, s8, 1
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_byte v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store i1 %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fp128_kernel_preload_arg(ptr addrspace(1) inreg %out, fp128 inreg %in) #0 {
+; GFX940-LABEL: fp128_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v3, s9
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: fp128_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s13
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store fp128 %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v7i8_kernel_preload_arg(ptr addrspace(1) inreg %out, <7 x i8> inreg %in) #0 {
+; GFX940-LABEL: v7i8_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    s_lshr_b32 s0, s5, 8
+; GFX940-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX940-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b32_e32 v3, s5
+; GFX940-NEXT:    v_or_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    global_store_byte_d16_hi v2, v3, s[2:3] offset:6 sc0 sc1
+; GFX940-NEXT:    global_store_short v2, v1, s[2:3] offset:4 sc0 sc1
+; GFX940-NEXT:    global_store_dword v2, v0, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v7i8_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 8
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    s_lshr_b32 s0, s9, 8
+; GFX90a-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s9
+; GFX90a-NEXT:    v_or_b32_sdwa v1, s9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    global_store_byte_d16_hi v2, v3, s[6:7] offset:6
+; GFX90a-NEXT:    global_store_short v2, v1, s[6:7] offset:4
+; GFX90a-NEXT:    global_store_dword v2, v0, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <7 x i8> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v7half_kernel_preload_arg(ptr addrspace(1) inreg %out, <7 x half> inreg %in) #0 {
+; GFX940-LABEL: v7half_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s9
+; GFX940-NEXT:    global_store_short v3, v0, s[2:3] offset:12 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v7half_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s13
+; GFX90a-NEXT:    global_store_short v3, v0, s[6:7] offset:12
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <7 x half> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @i16_i32_kernel_preload_arg(ptr addrspace(1) %out, i16 inreg %in, i32 inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: i16_i32_kernel_preload_arg:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x10
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v1, s6
+; GFX940-NEXT:    v_mov_b32_e32 v2, s7
+; GFX940-NEXT:    global_store_short v0, v1, s[4:5] sc0 sc1
+; GFX940-NEXT:    global_store_dword v0, v2, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i16_i32_kernel_preload_arg:
+; GFX90a:       ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s3
+; GFX90a-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX90a-NEXT:    global_store_dword v0, v2, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store i16 %in, ptr addrspace(1) %out
+  store i32 %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+define amdgpu_kernel void @i16_v3i32_kernel_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %in, <3 x i32> inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: i16_v3i32_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v4, s4
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    global_store_short v3, v4, s[2:3] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[10:11] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i16_v3i32_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x20
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v4, s8
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    global_store_short v3, v4, s[6:7]
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[0:1]
+; GFX90a-NEXT:    s_endpgm
+  store i16 %in, ptr addrspace(1) %out
+  store <3 x i32> %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+define amdgpu_kernel void @i16_i16_kernel_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %in, i16 inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: i16_i16_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    global_store_short_d16_hi v0, v1, s[6:7] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i16_i16_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_short_d16_hi v0, v1, s[10:11]
+; GFX90a-NEXT:    s_endpgm
+  store i16 %in, ptr addrspace(1) %out
+  store i16 %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+define amdgpu_kernel void @i16_v2i8_kernel_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %in, <2 x i8> inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: i16_v2i8_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0
+; GFX940-NEXT:    v_mov_b32_e32 v2, s4
+; GFX940-NEXT:    v_or_b32_sdwa v0, s0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    global_store_short v1, v2, s[2:3] sc0 sc1
+; GFX940-NEXT:    global_store_short v1, v0, s[6:7] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i16_v2i8_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    v_mov_b32_e32 v1, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s8
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    global_store_short v1, v2, s[6:7]
+; GFX90a-NEXT:    global_store_short v1, v0, s[10:11]
+; GFX90a-NEXT:    s_endpgm
+  store i16 %in, ptr addrspace(1) %out
+  store <2 x i8> %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+; The second argument is not expected to be preloaded with the current behavior.
+
+define amdgpu_kernel void @i32_ptr1_i32_staggered_preload_arg(i32 inreg %arg0, ptr addrspace(1) %out, i32 inreg %arg1) #0 {
+; GFX940-LABEL: i32_ptr1_i32_staggered_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dword s3, s[0:1], 0x10
+; GFX940-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_add_i32 s0, s2, s3
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i32_ptr1_i32_staggered_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dword s2, s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    s_add_i32 s2, s6, s2
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
+; GFX90a-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX90a-NEXT:    s_endpgm
+  %add = add i32 %arg0, %arg1
+  store i32 %add, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @ptr1_i8_trailing_unused(ptr addrspace(1) inreg %out, i8 inreg %arg0, i32 inreg %unused) #0 {
+; GFX940-LABEL: ptr1_i8_trailing_unused:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_and_b32 s0, s4, 0xff
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i8_trailing_unused:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_and_b32 s0, s8, 0xff
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  %ext = zext i8 %arg0 to i32
+  store i32 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+attributes #0 = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.mir b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.mir
new file mode 100644
index 000000000000000..17bed38bd046d74
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.mir
@@ -0,0 +1,120 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -run-pass=si-fold-operands -verify-machineinstrs -o - %s | FileCheck --check-prefix=GCN %s
+
+# Do not use inline constants for f16 pseudo scalar transcendentals.
+# But allow literal constants.
+
+---
+name: exp_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: exp_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_EXP_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_EXP_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_EXP_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: exp_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: exp_f16_literal
+    ; GCN: [[V_S_EXP_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_EXP_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_EXP_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: log_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: log_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_LOG_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_LOG_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_LOG_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: log_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: log_f16_literal
+    ; GCN: [[V_S_LOG_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_LOG_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_LOG_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: rcp_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: rcp_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_RCP_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_RCP_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_RCP_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: rcp_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: rcp_f16_literal
+    ; GCN: [[V_S_RCP_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_RCP_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_RCP_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: rsq_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: rsq_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_RSQ_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_RSQ_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_RSQ_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: rsq_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: rsq_f16_literal
+    ; GCN: [[V_S_RSQ_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_RSQ_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_RSQ_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: sqrt_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: sqrt_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_SQRT_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_SQRT_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_SQRT_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: sqrt_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: sqrt_f16_literal
+    ; GCN: [[V_S_SQRT_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_SQRT_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_SQRT_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
diff --git a/llvm/test/CodeGen/AMDGPU/scalarize-insert-subvector.ll b/llvm/test/CodeGen/AMDGPU/scalarize-insert-subvector.ll
new file mode 100644
index 000000000000000..f6f367cf85f7293
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/scalarize-insert-subvector.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefix=GCN %s
+
+define void @scalarize_insert_subvector(ptr addrspace(3) %inptr, ptr addrspace(3) %inptr1, ptr addrspace(3) %outptr) {
+; GCN-LABEL: scalarize_insert_subvector:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    ds_read_b64 v[4:5], v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_read_b32 v5, v1 offset:4
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_write_b64 v2, v[4:5]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %load0 = load <2 x i32>, ptr addrspace(3) %inptr, align 8
+  %load1= load <2 x i32>, ptr addrspace(3) %inptr1, align 8
+  %shuffle0 = shufflevector <2 x i32> %load1, <2 x i32> poison, <1 x i32> <i32 1>
+  %bitcast0 = bitcast <1 x i32> %shuffle0 to <2 x half>
+  %bitcast1 = bitcast <2 x i32> %load0 to <4 x half>
+  %shuffle1 = shufflevector <2 x half> %bitcast0, <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %shuffle2 = shufflevector <4 x half> %bitcast1, <4 x half> %shuffle1, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  store <4 x half> %shuffle2, ptr addrspace(3) %outptr
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/vopd-combine.mir b/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
index b8ac50c3aeb5e8f..8d5060177c63d6f 100644
--- a/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
+++ b/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
@@ -23,6 +23,8 @@
   define void @vopd_mov_fixup_fail() { ret void }
   define void @vopd_no_combine_dependent_subreg() { ret void }
   define void @vopd_mov_mov_same_src_bank() { ret void }
+  define void @vopd_combine_opy_overwrites_opx() { ret void }
+  define void @vopd_no_combine_opx_overwrites_opy() { ret void }
 ...
 
 ---
@@ -737,3 +739,50 @@ body:             |
     $vgpr2 = V_MOV_B32_e32 $vgpr1, implicit $exec
     $vgpr3 = V_MOV_B32_e32 $vgpr5, implicit $exec
 ...
+
+---
+# There is no race between mov and add, they are executed at the same time
+name:            vopd_combine_opy_overwrites_opx
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; SCHED-LABEL: name: vopd_combine_opy_overwrites_opx
+    ; SCHED: $vgpr1 = IMPLICIT_DEF
+    ; SCHED-NEXT: $vgpr3 = IMPLICIT_DEF
+    ; SCHED-NEXT: $vgpr0 = V_MOV_B32_e32 killed $vgpr1, implicit $exec
+    ; SCHED-NEXT: $vgpr1 = V_ADD_F32_e32 killed $vgpr3, $vgpr3, implicit $mode, implicit $exec
+    ;
+    ; PAIR-GFX11-LABEL: name: vopd_combine_opy_overwrites_opx
+    ; PAIR-GFX11: $vgpr1 = IMPLICIT_DEF
+    ; PAIR-GFX11-NEXT: $vgpr3 = IMPLICIT_DEF
+    ; PAIR-GFX11-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_ADD_F32_e32_gfx11 killed $vgpr1, killed $vgpr3, $vgpr3, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+    ;
+    ; PAIR-GFX12-LABEL: name: vopd_combine_opy_overwrites_opx
+    ; PAIR-GFX12: $vgpr1 = IMPLICIT_DEF
+    ; PAIR-GFX12-NEXT: $vgpr3 = IMPLICIT_DEF
+    ; PAIR-GFX12-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_ADD_F32_e32_gfx12 killed $vgpr1, killed $vgpr3, $vgpr3, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+    $vgpr1 = IMPLICIT_DEF
+    $vgpr3 = IMPLICIT_DEF
+    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
+    $vgpr1 = V_ADD_F32_e32 $vgpr3, $vgpr3, implicit $mode, implicit $exec
+...
+
+---
+# OPY cannot cosume value produced by OPX
+name:            vopd_no_combine_opx_overwrites_opy
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; SCHED-LABEL: name: vopd_no_combine_opx_overwrites_opy
+    ; SCHED: $vgpr1 = IMPLICIT_DEF
+    ; SCHED-NEXT: $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
+    ; SCHED-NEXT: $vgpr3 = V_ADD_F32_e32 killed $vgpr0, killed $vgpr1, implicit $mode, implicit $exec
+    ;
+    ; PAIR-LABEL: name: vopd_no_combine_opx_overwrites_opy
+    ; PAIR: $vgpr1 = IMPLICIT_DEF
+    ; PAIR-NEXT: $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
+    ; PAIR-NEXT: $vgpr3 = V_ADD_F32_e32 killed $vgpr0, killed $vgpr1, implicit $mode, implicit $exec
+    $vgpr1 = IMPLICIT_DEF
+    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
+    $vgpr3 = V_ADD_F32_e32 $vgpr0, $vgpr1, implicit $mode, implicit $exec
+...
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll b/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
index 411cf78b621f8bf..dc1d4b289c2ab81 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple arm-unknown -mattr=+vfp2,+v4t -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LITTLE
-; RUN: llc -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=BIG
+; RUN: llc -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -enable-arm-gisel-bigendian -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=BIG
 
 define void @test_void_return() {
 ; CHECK-LABEL: name: test_void_return
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll b/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
index e8cd182196b6221..65586f72c7c198b 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -O0 -mtriple arm-unknown -mattr=+vfp2,+v4t -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=LITTLE
-; RUN: llc -O0 -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=BIG
+; RUN: llc -O0 -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -enable-arm-gisel-bigendian -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=BIG
 ; RUN: llc -O0 -mtriple thumb-unknown -mattr=+vfp2,+v6t2 -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LITTLE -check-prefix=THUMB
 
 declare arm_aapcscc ptr @simple_reg_params_target(i32, ptr)
diff --git a/llvm/test/CodeGen/BPF/sockex2.ll b/llvm/test/CodeGen/BPF/sockex2.ll
index 4131d9dac31d88b..b1264099f64c60b 100644
--- a/llvm/test/CodeGen/BPF/sockex2.ll
+++ b/llvm/test/CodeGen/BPF/sockex2.ll
@@ -311,7 +311,7 @@ flow_dissector.exit.thread:                       ; preds = %86, %12, %196, %199
 ; CHECK-LABEL: bpf_prog2:
 ; CHECK: r0 = *(u16 *)skb[12] # encoding: [0x28,0x00,0x00,0x00,0x0c,0x00,0x00,0x00]
 ; CHECK: r0 = *(u16 *)skb[16] # encoding: [0x28,0x00,0x00,0x00,0x10,0x00,0x00,0x00]
-; CHECK: implicit-def: $r8
+; CHECK: implicit-def: $r7
 ; CHECK: r1 =
 ; CHECK: call 1 # encoding: [0x85,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
 ; CHECK: call 2 # encoding: [0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00]
diff --git a/llvm/test/CodeGen/DirectX/conflicting-bitcast-insert.ll b/llvm/test/CodeGen/DirectX/conflicting-bitcast-insert.ll
index 01084f285cd72ac..8f5d3ae8641717f 100644
--- a/llvm/test/CodeGen/DirectX/conflicting-bitcast-insert.ll
+++ b/llvm/test/CodeGen/DirectX/conflicting-bitcast-insert.ll
@@ -7,7 +7,7 @@ define i64 @test(ptr %p) {
   ret i64 %v
 }
 
-; CHECK: define i64 @test(ptr %p) {
+; CHECK: define internal i64 @test(ptr %p) {
 ; CHECK-NEXT: %1 = bitcast ptr %p to ptr
 ; CHECK-NEXT: store i32 0, ptr %1, align 4
 ; CHECK-NEXT: %2 = bitcast ptr %p to ptr
@@ -19,7 +19,7 @@ define i64 @testGEP(ptr %p) {
   ret i64 %val
 }
 
-; CHECK: define i64 @testGEP(ptr %p) {
+; CHECK: define internal i64 @testGEP(ptr %p) {
 ; CHECK-NEXT:   %1 = bitcast ptr %p to ptr
 ; CHECK-NEXT:   %ptr = getelementptr i32, ptr %1, i32 4
 ; CHECK-NEXT:   %2 = bitcast ptr %p to ptr
diff --git a/llvm/test/CodeGen/DirectX/finalize_linkage.ll b/llvm/test/CodeGen/DirectX/finalize_linkage.ll
new file mode 100644
index 000000000000000..0ee8a5f44593ba1
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/finalize_linkage.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -dxil-finalize-linkage -mtriple=dxil-unknown-shadermodel6.5-compute %s | FileCheck %s
+; RUN: llc %s --filetype=asm -o - | FileCheck %s --check-prefixes=CHECK-LLC
+
+target triple = "dxilv1.5-pc-shadermodel6.5-compute"
+
+; DXILFinalizeLinkage changes linkage of all functions that are not
+; entry points or exported function to internal.
+
+; CHECK: define internal void @"?f1@@YAXXZ"()
+define void @"?f1@@YAXXZ"() #0 {
+entry:
+  ret void
+}
+
+; CHECK: define internal void @"?f2@@YAXXZ"()
+define void @"?f2@@YAXXZ"() #0 {
+entry:
+  ret void
+}
+
+; CHECK: define internal void @"?f3@@YAXXZ"()
+define void @"?f3@@YAXXZ"() #0 {
+entry:
+  ret void
+}
+
+; CHECK: define internal void @"?foo@@YAXXZ"()
+define void @"?foo@@YAXXZ"() #0 {
+entry:
+  call void @"?f2@@YAXXZ"() #3
+  ret void
+}
+
+; Exported function - do not change linkage
+; CHECK: define void @"?bar@@YAXXZ"()
+define void @"?bar@@YAXXZ"() #1 {
+entry:
+  call void @"?f3@@YAXXZ"() #3
+  ret void
+}
+
+; CHECK: define internal void @"?main@@YAXXZ"() #0
+define internal void @"?main@@YAXXZ"() #0 {
+entry:
+  call void @"?foo@@YAXXZ"() #3
+  call void @"?bar@@YAXXZ"() #3
+  ret void
+}
+
+; Entry point function - do not change linkage
+; CHECK: define void @main() #2
+define void @main() #2 {
+entry:
+  call void @"?main@@YAXXZ"()
+  ret void
+}
+
+attributes #0 = { convergent noinline nounwind optnone}
+attributes #1 = { convergent noinline nounwind optnone "hlsl.export"}
+attributes #2 = { convergent "hlsl.numthreads"="4,1,1" "hlsl.shader"="compute"}
+attributes #3 = { convergent }
+
+; Make sure "hlsl.export" attribute is stripped by llc
+; CHECK-LLC-NOT: "hlsl.export"
diff --git a/llvm/test/CodeGen/DirectX/omit-bitcast-insert.ll b/llvm/test/CodeGen/DirectX/omit-bitcast-insert.ll
index 73c33e8cfdd8648..6066a0033e45dee 100644
--- a/llvm/test/CodeGen/DirectX/omit-bitcast-insert.ll
+++ b/llvm/test/CodeGen/DirectX/omit-bitcast-insert.ll
@@ -6,7 +6,7 @@ define i64 @test(ptr %p) {
   ret i64 %v
 }
 
-; CHECK: define i64 @test(ptr %p) {
+; CHECK: define internal i64 @test(ptr %p) {
 ; CHECK-NEXT: %v = load i64, ptr %p, align 8
 ; CHECK-NEXT: ret i64 %v
 
@@ -16,7 +16,7 @@ define i64 @test2(ptr %p) {
   ret i64 %v
 }
 
-; CHECK: define i64 @test2(ptr %p) {
+; CHECK: define internal i64 @test2(ptr %p) {
 ; CHECK-NEXT: store i64 0, ptr %p
 ; CHECK-NEXT: %v = load i64, ptr %p, align 8
 ; CHECK-NEXT: ret i64 %v
@@ -27,6 +27,6 @@ define i32 @test3(ptr %0)  {
   ret i32 %3
 }
 
-; CHECK: define i32 @test3(ptr %0)  {
+; CHECK: define internal i32 @test3(ptr %0)  {
 ; CHECK-NEXT: %2 = getelementptr i32, ptr %0, i32 4
 ; CHECK-NEXT: %3 = load i32, ptr %2
diff --git a/llvm/test/CodeGen/Hexagon/block-addr.ll b/llvm/test/CodeGen/Hexagon/block-addr.ll
index cbe824643aca28a..5d028bba946407d 100644
--- a/llvm/test/CodeGen/Hexagon/block-addr.ll
+++ b/llvm/test/CodeGen/Hexagon/block-addr.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
 
 ; CHECK-DAG: r[[REG:[0-9]+]] = memw(r{{[0-9]+<<#[0-9]+}}+##.LJTI{{.*}})
 ; CHECK-DAG: jumpr r[[REG]]
diff --git a/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals.mir b/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals.mir
new file mode 100644
index 000000000000000..7fa3cdf62d09025
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals.mir
@@ -0,0 +1,217 @@
+# REQUIRES: asserts
+# 
+# RUN: llc --march=hexagon %s -run-pass=pipeliner -debug-only=pipeliner \
+# RUN: -window-sched=force -filetype=null -window-search-num=100 \
+# RUN: -window-search-ratio=100 -window-diff-limit=0 -verify-machineinstrs \
+# RUN: 2>&1 | FileCheck %s
+
+# The bug was reported at https://github.com/llvm/llvm-project/pull/99454. 
+# It is caused by the corruption of live intervals in certain scenarios.
+#
+# We check the newly generated MBBs after successful scheduling here.
+# CHECK: Best window offset is {{[0-9]+}} and Best II is {{[0-9]+}}.
+# CHECK: prolog:
+# CHECK: bb.5:
+# CHECK: New block
+# CHECK: bb.6:
+# CHECK: epilog:
+# CHECK: bb.7:
+# CHECK: Best window offset is {{[0-9]+}} and Best II is {{[0-9]+}}.
+# CHECK: prolog:
+# CHECK: bb.8:
+# CHECK: New block
+# CHECK: bb.9:
+# CHECK: epilog:
+# CHECK: bb.10:
+
+--- |
+  target triple = "hexagon"
+  
+  @_dp_ctrl_calc_tu_temp2_fp = global i64 0
+  @_dp_ctrl_calc_tu_temp1_fp = global i32 0
+  @dp_panel_update_tu_timings___trans_tmp_5 = global i64 0
+  @_dp_ctrl_calc_tu___trans_tmp_8 = global i64 0
+  
+  declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+  declare i8 @div64_u64_rem(i32, ptr)
+  declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+  
+  define void @dp_ctrl_calc_tu_parameters() {
+  if.end.i:
+    %rem.i11.i = alloca i64, align 8
+    %rem.i.i = alloca i64, align 8
+    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %rem.i11.i)
+    %call.i.i = call i8 @div64_u64_rem(i32 0, ptr nonnull %rem.i11.i)
+    %conv1.i.i = zext i8 %call.i.i to i64
+    %rem.promoted.i.i = load i64, ptr %rem.i11.i, align 8
+    br label %do.body.i.i
+  
+  do.body.i.i:
+    %lsr.iv1 = phi i32 [ %lsr.iv.next2, %do.body.i.i ], [ -32, %if.end.i ]
+    %sub9.i.i = phi i64 [ %rem.promoted.i.i, %if.end.i ], [ %sub8.i.i.7, %do.body.i.i ]
+    %res_abs.0.i.i = phi i64 [ %conv1.i.i, %if.end.i ], [ %res_abs.1.i.i.7, %do.body.i.i ]
+    %cmp.not.i.i = icmp ne i64 %sub9.i.i, 0
+    %sub.i.neg.i = sext i1 %cmp.not.i.i to i64
+    %sub8.i.i = add i64 %sub9.i.i, %sub.i.neg.i
+    %0 = shl i64 %res_abs.0.i.i, 2
+    %1 = select i1 %cmp.not.i.i, i64 2, i64 0
+    %shl.i.i.5 = or disjoint i64 %0, %1
+    %cmp.not.i.i.5 = icmp ne i64 %sub8.i.i, 0
+    %sub.i.neg.i.5 = sext i1 %cmp.not.i.i.5 to i64
+    %sub8.i.i.5 = add i64 %sub8.i.i, %sub.i.neg.i.5
+    %or.i.i.5 = zext i1 %cmp.not.i.i.5 to i64
+    %res_abs.1.i.i.5 = or disjoint i64 %shl.i.i.5, %or.i.i.5
+    %cmp.not.i.i.6 = icmp ne i64 %sub8.i.i.5, 0
+    %sub.i.neg.i.6 = sext i1 %cmp.not.i.i.6 to i64
+    %sub8.i.i.6 = add i64 %sub8.i.i.5, %sub.i.neg.i.6
+    %2 = shl i64 %res_abs.1.i.i.5, 2
+    %3 = select i1 %cmp.not.i.i.6, i64 2, i64 0
+    %shl.i.i.7 = or disjoint i64 %2, %3
+    %cmp.not.i.i.7 = icmp ne i64 %sub8.i.i.6, 0
+    %sub.i.neg.i.7 = sext i1 %cmp.not.i.i.7 to i64
+    %sub8.i.i.7 = add i64 %sub8.i.i.6, %sub.i.neg.i.7
+    %or.i.i.7 = zext i1 %cmp.not.i.i.7 to i64
+    %res_abs.1.i.i.7 = or disjoint i64 %shl.i.i.7, %or.i.i.7
+    %lsr.iv.next2 = add nsw i32 %lsr.iv1, 8
+    %tobool.not.i.i.7 = icmp eq i32 %lsr.iv.next2, 0
+    br i1 %tobool.not.i.i.7, label %fec_check.i, label %do.body.i.i
+  
+  fec_check.i:
+    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %rem.i11.i)
+    store i64 %res_abs.1.i.i.7, ptr @_dp_ctrl_calc_tu_temp2_fp, align 8
+    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %rem.i11.i)
+    %call.i12.i = call i8 @div64_u64_rem(i32 0, ptr nonnull %rem.i11.i)
+    %conv1.i13.i = zext i8 %call.i12.i to i64
+    %rem.promoted.i14.i = load i64, ptr %rem.i11.i, align 8
+    br label %do.body.i15.i
+  
+  do.body.i15.i:
+    %lsr.iv = phi i32 [ %lsr.iv.next, %do.body.i15.i ], [ -32, %fec_check.i ]
+    %sub9.i16.i = phi i64 [ %rem.promoted.i14.i, %fec_check.i ], [ %sub8.i22.i.7, %do.body.i15.i ]
+    %res_abs.0.i17.i = phi i64 [ %conv1.i13.i, %fec_check.i ], [ %res_abs.1.i24.i.7, %do.body.i15.i ]
+    %cmp.not.i20.i = icmp ugt i64 %sub9.i16.i, 999
+    %sub.i21.neg.i = select i1 %cmp.not.i20.i, i64 -1000, i64 0
+    %sub8.i22.i = add i64 %sub.i21.neg.i, %sub9.i16.i
+    %4 = shl i64 %res_abs.0.i17.i, 2
+    %5 = select i1 %cmp.not.i20.i, i64 2, i64 0
+    %shl.i19.i.7 = or disjoint i64 %4, %5
+    %cmp.not.i20.i.7 = icmp ugt i64 %sub8.i22.i, 999
+    %sub.i21.neg.i.7 = select i1 %cmp.not.i20.i.7, i64 -1000, i64 0
+    %sub8.i22.i.7 = add i64 %sub.i21.neg.i.7, %sub8.i22.i
+    %or.i23.i.7 = zext i1 %cmp.not.i20.i.7 to i64
+    %res_abs.1.i24.i.7 = or disjoint i64 %shl.i19.i.7, %or.i23.i.7
+    %lsr.iv.next = add nsw i32 %lsr.iv, 8
+    %tobool.not.i26.i.7 = icmp eq i32 %lsr.iv.next, 0
+    br i1 %tobool.not.i26.i.7, label %_dp_ctrl_calc_tu.exit, label %do.body.i15.i
+  
+  _dp_ctrl_calc_tu.exit:
+    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %rem.i11.i)
+    %conv.i = trunc i64 %res_abs.1.i24.i.7 to i32
+    store i32 %conv.i, ptr @_dp_ctrl_calc_tu_temp1_fp, align 4
+    %conv5.i = and i64 %res_abs.1.i24.i.7, 4294967295
+    store i64 %conv5.i, ptr @dp_panel_update_tu_timings___trans_tmp_5, align 8
+    store i64 %res_abs.1.i.i.7, ptr @_dp_ctrl_calc_tu___trans_tmp_8, align 8
+    ret void
+  }
+
+...
+---
+name:            dp_ctrl_calc_tu_parameters
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: rem.i11.i, type: default, offset: 0, size: 8, alignment: 8}
+body:             |
+  bb.0:
+    successors: %bb.1(0x80000000)
+  
+    %0:intregs = A2_tfrsi 0
+    %1:intregs = PS_fi %stack.0.rem.i11.i, 0
+    %2:intregs = A2_tfrsi 0
+    %3:doubleregs = A4_combineir 0, %2
+    %4:doubleregs = L2_loadrd_io %stack.0.rem.i11.i, 0
+    %5:doubleregs = A2_tfrpi 0
+    J2_loop0i %bb.1, 4, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+  
+  bb.1 (machine-block-address-taken):
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  
+    %6:doubleregs = PHI %4, %bb.0, %7, %bb.1
+    %8:doubleregs = PHI %3, %bb.0, %9, %bb.1
+    %10:predregs = C2_cmpeqp %6, %5
+    %11:intregs = C2_muxii %10, 0, -1
+    %12:doubleregs = A2_addsp %11, %6
+    %13:doubleregs = S2_asl_i_p %8, 2
+    %14:intregs = S2_setbit_i %13.isub_lo, 1
+    %15:intregs = C2_mux %10, %13.isub_lo, %14
+    %16:predregs = C2_cmpeqp %12, %5
+    %17:intregs = C2_muxii %16, 0, -1
+    %18:doubleregs = A2_addsp %17, %12
+    %19:intregs = S2_setbit_i %15, 0
+    %20:intregs = C2_mux %16, %15, %19
+    %21:predregs = C2_cmpeqp %18, %5
+    %22:intregs = C2_muxii %21, 0, -1
+    %23:doubleregs = A2_addsp %22, %18
+    %24:intregs = S2_asl_i_r %20, 2
+    %25:intregs = S2_extractu %8.isub_lo, 2, 28
+    %26:intregs = S2_asl_i_r_or %25, %13.isub_hi, 2
+    %27:intregs = S2_setbit_i %24, 1
+    %28:intregs = C2_mux %21, %24, %27
+    %29:predregs = C2_cmpeqp %23, %5
+    %30:intregs = C2_muxii %29, 0, -1
+    %7:doubleregs = A2_addsp %30, %23
+    %31:intregs = S2_setbit_i %28, 0
+    %32:intregs = C2_mux %29, %28, %31
+    %9:doubleregs = REG_SEQUENCE %26, %subreg.isub_hi, %32, %subreg.isub_lo
+    ENDLOOP0 %bb.1, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def dead $pc
+  
+  bb.2:
+    successors: %bb.3(0x80000000)
+  
+    S2_storerdgp @_dp_ctrl_calc_tu_temp2_fp, %9, implicit $gp
+    %33:intregs = A2_tfrsi 0
+    %34:intregs = PS_fi %stack.0.rem.i11.i, 0
+    %35:intregs = A2_tfrsi 0
+    %36:doubleregs = L2_loadrd_io %stack.0.rem.i11.i, 0
+    %37:doubleregs = A2_tfrpi 124
+    %38:intregs = A2_tfrsi -1000
+    %39:intregs = A2_tfrsi -1
+    J2_loop0i %bb.3, 4, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+  
+  bb.3 (machine-block-address-taken):
+    successors: %bb.4(0x04000000), %bb.3(0x7c000000)
+  
+    %40:doubleregs = PHI %36, %bb.2, %41, %bb.3
+    %42:intregs = PHI %35, %bb.2, %43, %bb.3
+    %44:intregs = PHI %33, %bb.2, %45, %bb.3
+    %46:doubleregs = S2_lsr_i_p %40, 3
+    %47:predregs = C2_cmpgtup %46, %37
+    %48:intregs = C2_mux %47, %38, %33
+    %49:intregs = C2_mux %47, %39, %33
+    %50:doubleregs = REG_SEQUENCE %49, %subreg.isub_hi, %48, %subreg.isub_lo
+    %51:doubleregs = A2_addp %50, %40
+    %52:intregs = S2_asl_i_r %42, 2
+    %53:intregs = S2_extractu %42, 2, 30
+    %45:intregs = S2_asl_i_r_or %53, %44, 2
+    %54:intregs = S2_setbit_i %52, 1
+    %55:intregs = C2_mux %47, %54, %52
+    %56:doubleregs = S2_lsr_i_p %51, 3
+    %57:predregs = C2_cmpgtup %56, %37
+    %58:intregs = C2_mux %57, %38, %33
+    %59:intregs = C2_mux %57, %39, %33
+    %60:doubleregs = REG_SEQUENCE %59, %subreg.isub_hi, %58, %subreg.isub_lo
+    %41:doubleregs = A2_addp %60, %51
+    %61:intregs = S2_setbit_i %55, 0
+    %43:intregs = C2_mux %57, %61, %55
+    ENDLOOP0 %bb.3, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.4, implicit-def dead $pc
+  
+  bb.4:
+    S2_storerigp @_dp_ctrl_calc_tu_temp1_fp, %43, implicit $gp
+    %62:intregs = A2_tfrsi 0
+    %63:doubleregs = REG_SEQUENCE %43, %subreg.isub_lo, %62, %subreg.isub_hi
+    S2_storerdgp @dp_panel_update_tu_timings___trans_tmp_5, %63, implicit $gp
+    S2_storerdgp @_dp_ctrl_calc_tu___trans_tmp_8, %9, implicit $gp
+    PS_jmpret $r31, implicit-def dead $pc
+
+...
diff --git a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
index 49155a3966a8429..6d5cf5a94293120 100644
--- a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
+++ b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
@@ -17,20 +17,16 @@ define dso_local noundef signext i32 @main() nounwind {
 ; CHECK-NEXT:    addi.d $sp, $sp, -272
 ; CHECK-NEXT:    st.d $ra, $sp, 264 # 8-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvld $xr0, $a0, %pc_lo12(.LCPI0_0)
 ; CHECK-NEXT:    xvst $xr0, $sp, 96 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_1)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_1)
-; CHECK-NEXT:    xvld $xr1, $a0, 0
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI0_1)
 ; CHECK-NEXT:    xvst $xr1, $sp, 64 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_2)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_2)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_2)
 ; CHECK-NEXT:    xvst $xr2, $sp, 32 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_3)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_3)
-; CHECK-NEXT:    xvld $xr3, $a0, 0
+; CHECK-NEXT:    xvld $xr3, $a0, %pc_lo12(.LCPI0_3)
 ; CHECK-NEXT:    xvst $xr3, $sp, 0 # 32-byte Folded Spill
 ; CHECK-NEXT:    xvst $xr0, $sp, 136
 ; CHECK-NEXT:    xvst $xr1, $sp, 168
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
index c51fded410e83b0..1af2b38d7994363 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
@@ -72,6 +72,22 @@ define i64 @load_acquire_i64(ptr %ptr) {
   ret i64 %val
 }
 
+define ptr @load_acquire_ptr(ptr %ptr) {
+; LA32-LABEL: load_acquire_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    dbar 20
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_acquire_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:    ret
+  %val = load atomic ptr, ptr %ptr acquire, align 8
+  ret ptr %val
+}
+
 define i8 @load_unordered_i8(ptr %ptr) {
 ; LA32-LABEL: load_unordered_i8:
 ; LA32:       # %bb.0:
@@ -135,6 +151,20 @@ define i64 @load_unordered_i64(ptr %ptr) {
   ret i64 %val
 }
 
+define ptr @load_unordered_ptr(ptr %ptr) {
+; LA32-LABEL: load_unordered_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_unordered_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic ptr, ptr %ptr unordered, align 8
+  ret ptr %val
+}
+
 define i8 @load_monotonic_i8(ptr %ptr) {
 ; LA32-LABEL: load_monotonic_i8:
 ; LA32:       # %bb.0:
@@ -198,6 +228,20 @@ define i64 @load_monotonic_i64(ptr %ptr) {
   ret i64 %val
 }
 
+define ptr @load_monotonic_ptr(ptr %ptr) {
+; LA32-LABEL: load_monotonic_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_monotonic_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic ptr, ptr %ptr monotonic, align 8
+  ret ptr %val
+}
+
 define i8 @load_seq_cst_i8(ptr %ptr) {
 ; LA32-LABEL: load_seq_cst_i8:
 ; LA32:       # %bb.0:
@@ -268,6 +312,22 @@ define i64 @load_seq_cst_i64(ptr %ptr) {
   ret i64 %val
 }
 
+define ptr @load_seq_cst_ptr(ptr %ptr) {
+; LA32-LABEL: load_seq_cst_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_seq_cst_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    dbar 16
+; LA64-NEXT:    ret
+  %val = load atomic ptr, ptr %ptr seq_cst, align 8
+  ret ptr %val
+}
+
 define void @store_release_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_release_i8:
 ; LA32:       # %bb.0:
@@ -336,6 +396,21 @@ define void @store_release_i64(ptr %ptr, i64 %v) {
   ret void
 }
 
+define void @store_release_ptr(ptr %ptr, ptr %v) {
+; LA32-LABEL: store_release_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    dbar 18
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: store_release_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.d $zero, $a1, $a0
+; LA64-NEXT:    ret
+  store atomic ptr %v, ptr %ptr release, align 8
+  ret void
+}
+
 define void @store_unordered_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_unordered_i8:
 ; LA32:       # %bb.0:
@@ -399,6 +474,20 @@ define void @store_unordered_i64(ptr %ptr, i64 %v) {
   ret void
 }
 
+define void @store_unordered_ptr(ptr %ptr, ptr %v) {
+; LA32-LABEL: store_unordered_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: store_unordered_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    st.d $a1, $a0, 0
+; LA64-NEXT:    ret
+  store atomic ptr %v, ptr %ptr unordered, align 8
+  ret void
+}
+
 define void @store_monotonic_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_monotonic_i8:
 ; LA32:       # %bb.0:
@@ -462,6 +551,20 @@ define void @store_monotonic_i64(ptr %ptr, i64 %v) {
   ret void
 }
 
+define void @store_monotonic_ptr(ptr %ptr, ptr %v) {
+; LA32-LABEL: store_monotonic_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: store_monotonic_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    st.d $a1, $a0, 0
+; LA64-NEXT:    ret
+  store atomic ptr %v, ptr %ptr monotonic, align 8
+  ret void
+}
+
 define void @store_seq_cst_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_seq_cst_i8:
 ; LA32:       # %bb.0:
@@ -534,3 +637,19 @@ define void @store_seq_cst_i64(ptr %ptr, i64 %v) {
   store atomic i64 %v, ptr %ptr seq_cst, align 8
   ret void
 }
+
+define void @store_seq_cst_ptr(ptr %ptr, ptr %v) {
+; LA32-LABEL: store_seq_cst_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: store_seq_cst_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.d $zero, $a1, $a0
+; LA64-NEXT:    ret
+  store atomic ptr %v, ptr %ptr seq_cst, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
index 3a0fd0758cb324f..b06f6523e977c5f 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
@@ -153,8 +153,7 @@ define void @buildvector_v32i8_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v32i8_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI12_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI12_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI12_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -166,8 +165,7 @@ define void @buildvector_v16i16_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v16i16_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI13_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI13_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI13_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -179,8 +177,7 @@ define void @buildvector_v8i32_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v8i32_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI14_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI14_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI14_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -192,8 +189,7 @@ define void @buildvector_v4i64_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v4i64_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI15_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI15_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI15_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -205,8 +201,7 @@ define void @buildvector_v2f32_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v2f32_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI16_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI16_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI16_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -218,8 +213,7 @@ define void @buildvector_v4f64_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v4f64_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI17_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI17_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI17_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shuffle-as-xvshuf.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shuffle-as-xvshuf.ll
index 4cc819018f0a8d1..6a8880514871572 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shuffle-as-xvshuf.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shuffle-as-xvshuf.ll
@@ -6,8 +6,7 @@ define <32 x i8> @shufflevector_v32i8(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK-LABEL: shufflevector_v32i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_0)
 ; CHECK-NEXT:    xvshuf.b $xr0, $xr1, $xr0, $xr2
 ; CHECK-NEXT:    ret
     %c = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 8, i32 10, i32 12, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
@@ -20,8 +19,7 @@ define <16 x i16> @shufflevector_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: shufflevector_v16i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI1_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI1_0)
 ; CHECK-NEXT:    xvpermi.d $xr0, $xr0, 78
 ; CHECK-NEXT:    xvpermi.d $xr1, $xr1, 78
 ; CHECK-NEXT:    xvshuf.h $xr2, $xr1, $xr0
@@ -37,8 +35,7 @@ define <8 x i32> @shufflevector_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: shufflevector_v8i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI2_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI2_0)
 ; CHECK-NEXT:    xvpermi.d $xr0, $xr0, 68
 ; CHECK-NEXT:    xvpermi.d $xr1, $xr1, 68
 ; CHECK-NEXT:    xvshuf.w $xr2, $xr1, $xr0
@@ -53,8 +50,7 @@ define <4 x i64> @shufflevector_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-LABEL: shufflevector_v4i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI3_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI3_0)
 ; CHECK-NEXT:    xvpermi.d $xr0, $xr0, 238
 ; CHECK-NEXT:    xvpermi.d $xr1, $xr1, 238
 ; CHECK-NEXT:    xvshuf.d $xr2, $xr1, $xr0
@@ -69,8 +65,7 @@ define <8 x float> @shufflevector_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: shufflevector_v8f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI4_0)
 ; CHECK-NEXT:    xvshuf.w $xr2, $xr1, $xr0
 ; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vselect.ll b/llvm/test/CodeGen/LoongArch/lasx/vselect.ll
index 7b4c7ced4b5f363..17ba28afc81f5f9 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/vselect.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/vselect.ll
@@ -72,8 +72,7 @@ define void @select_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind {
 ; CHECK-NEXT:    xvld $xr0, $a1, 0
 ; CHECK-NEXT:    xvld $xr1, $a2, 0
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    xvld $xr2, $a1, 0
+; CHECK-NEXT:    xvld $xr2, $a1, %pc_lo12(.LCPI4_0)
 ; CHECK-NEXT:    xvbitsel.v $xr0, $xr1, $xr0, $xr2
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll
index c04d7ca889f7ef6..ba19fe75d7570c9 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll
@@ -153,8 +153,7 @@ define void @buildvector_v16i8_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v16i8_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI12_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI12_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI12_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -166,8 +165,7 @@ define void @buildvector_v8i16_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v8i16_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI13_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI13_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI13_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -179,8 +177,7 @@ define void @buildvector_v4i32_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v4i32_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI14_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI14_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI14_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -192,8 +189,7 @@ define void @buildvector_v2i64_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v2i64_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI15_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI15_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI15_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -205,8 +201,7 @@ define void @buildvector_v2f32_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v2f32_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI16_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI16_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI16_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -218,8 +213,7 @@ define void @buildvector_v2f64_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v2f64_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI17_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI17_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI17_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shuffle-as-vshuf.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shuffle-as-vshuf.ll
index 55800b31446b3d8..ac78a26ba43673f 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shuffle-as-vshuf.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shuffle-as-vshuf.ll
@@ -5,8 +5,7 @@ define <16 x i8> @shufflevector_v16i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: shufflevector_v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI0_0)
 ; CHECK-NEXT:    vshuf.b $vr0, $vr1, $vr0, $vr2
 ; CHECK-NEXT:    ret
     %c = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 8, i32 10, i32 12, i32 15, i32 2, i32 4, i32 6, i32 8, i32 25, i32 30, i32 31, i32 31>
@@ -18,8 +17,7 @@ define <8 x i16> @shufflevector_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: shufflevector_v8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI1_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI1_0)
 ; CHECK-NEXT:    vshuf.h $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
@@ -32,8 +30,7 @@ define <4 x i32> @shufflevector_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: shufflevector_v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI2_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI2_0)
 ; CHECK-NEXT:    vshuf.w $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
@@ -46,8 +43,7 @@ define <2 x i64> @shufflevector_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: shufflevector_v2i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI3_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI3_0)
 ; CHECK-NEXT:    vshuf.d $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
@@ -60,8 +56,7 @@ define <4 x float> @shufflevector_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: shufflevector_v4f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI4_0)
 ; CHECK-NEXT:    vshuf.w $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
@@ -74,8 +69,7 @@ define <2 x double> @shufflevector_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: shufflevector_v2f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI5_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI5_0)
 ; CHECK-NEXT:    vshuf.d $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
index 48ef3c14a4bf513..1c10e6c3087ad76 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
@@ -72,8 +72,7 @@ define void @select_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind {
 ; CHECK-NEXT:    vld $vr0, $a1, 0
 ; CHECK-NEXT:    vld $vr1, $a2, 0
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    vld $vr2, $a1, 0
+; CHECK-NEXT:    vld $vr2, $a1, %pc_lo12(.LCPI4_0)
 ; CHECK-NEXT:    vbitsel.v $vr0, $vr1, $vr0, $vr2
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/merge-base-offset.ll b/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
index 32a4c4bdd1508a9..1e7a79beb62c610 100644
--- a/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
+++ b/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --mattr=+d --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 --mattr=+lasx --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefix=LA32 %s
-; RUN: llc --mtriple=loongarch64 --mattr=+d --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefix=LA64 %s
-; RUN: llc --mtriple=loongarch64 --mattr=+d --verify-machineinstrs \
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx --verify-machineinstrs \
 ; RUN:   --code-model=large < %s | FileCheck --check-prefix=LA64-LARGE %s
 
 @g_i8 = dso_local global i8 0
@@ -405,10 +405,7 @@ define dso_local void @store_f64() nounwind {
 ; LA32-LABEL: store_f64:
 ; LA32:       # %bb.0: # %entry
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_f64)
-; LA32-NEXT:    addi.w $a1, $zero, 1
-; LA32-NEXT:    movgr2fr.w $fa0, $a1
-; LA32-NEXT:    ffint.s.w $fa0, $fa0
-; LA32-NEXT:    fcvt.d.s $fa0, $fa0
+; LA32-NEXT:    vldi $vr0, -912
 ; LA32-NEXT:    fst.d $fa0, $a0, %pc_lo12(g_f64)
 ; LA32-NEXT:    ret
 ;
@@ -538,6 +535,162 @@ entry:
   ret void
 }
 
+@g_i32x4_src = dso_local global [4 x i32] zeroinitializer, align 16
+@g_i32x4_dst = dso_local global [4 x i32] zeroinitializer, align 16
+
+define dso_local void @copy_i32x4() nounwind {
+; LA32-LABEL: copy_i32x4:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA32-NEXT:    vld $vr0, $a0, %pc_lo12(g_i32x4_src)
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA32-NEXT:    vst $vr0, $a0, %pc_lo12(g_i32x4_dst)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: copy_i32x4:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA64-NEXT:    vld $vr0, $a0, %pc_lo12(g_i32x4_src)
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA64-NEXT:    vst $vr0, $a0, %pc_lo12(g_i32x4_dst)
+; LA64-NEXT:    ret
+;
+; LA64-LARGE-LABEL: copy_i32x4:
+; LA64-LARGE:       # %bb.0: # %entry
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_src)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_src)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_src)
+; LA64-LARGE-NEXT:    vldx $vr0, $a1, $a0
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_dst)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    vstx $vr0, $a1, $a0
+; LA64-LARGE-NEXT:    ret
+entry:
+  %0 = load <4 x i32>, ptr @g_i32x4_src, align 16
+  store <4 x i32> %0, ptr @g_i32x4_dst, align 16
+  ret void
+}
+
+@g_i32x8_src = dso_local global [8 x i32] zeroinitializer, align 32
+@g_i32x8_dst = dso_local global [8 x i32] zeroinitializer, align 32
+
+define dso_local void @copy_i32x8() nounwind {
+; LA32-LABEL: copy_i32x8:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA32-NEXT:    xvld $xr0, $a0, %pc_lo12(g_i32x4_src)
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA32-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i32x4_dst)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: copy_i32x8:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA64-NEXT:    xvld $xr0, $a0, %pc_lo12(g_i32x4_src)
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA64-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i32x4_dst)
+; LA64-NEXT:    ret
+;
+; LA64-LARGE-LABEL: copy_i32x8:
+; LA64-LARGE:       # %bb.0: # %entry
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_src)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_src)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_src)
+; LA64-LARGE-NEXT:    xvldx $xr0, $a1, $a0
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_dst)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    xvstx $xr0, $a1, $a0
+; LA64-LARGE-NEXT:    ret
+entry:
+  %0 = load <8 x i32>, ptr @g_i32x4_src, align 32
+  store <8 x i32> %0, ptr @g_i32x4_dst, align 32
+  ret void
+}
+
+@g_i8x16 = dso_local global <16 x i8> zeroinitializer, align 16
+
+define void @copy_i8_to_i8x16() {
+; LA32-LABEL: copy_i8_to_i8x16:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA32-NEXT:    vldrepl.b $vr0, $a0, %pc_lo12(g_i8)
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x16)
+; LA32-NEXT:    vst $vr0, $a0, %pc_lo12(g_i8x16)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: copy_i8_to_i8x16:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA64-NEXT:    vldrepl.b $vr0, $a0, %pc_lo12(g_i8)
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x16)
+; LA64-NEXT:    vst $vr0, $a0, %pc_lo12(g_i8x16)
+; LA64-NEXT:    ret
+;
+; LA64-LARGE-LABEL: copy_i8_to_i8x16:
+; LA64-LARGE:       # %bb.0: # %entry
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    vldrepl.b $vr0, $a0, 0
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x16)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8x16)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8x16)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8x16)
+; LA64-LARGE-NEXT:    vstx $vr0, $a1, $a0
+; LA64-LARGE-NEXT:    ret
+entry:
+  %0 = call <16 x i8> @llvm.loongarch.lsx.vldrepl.b(ptr @g_i8, i32 0)
+  store <16 x i8> %0, ptr @g_i8x16, align 16
+  ret void
+}
+
+@g_i8x32 = dso_local global <32 x i8> zeroinitializer, align 32
+
+define void @copy_i8_to_i8x32() {
+; LA32-LABEL: copy_i8_to_i8x32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA32-NEXT:    xvldrepl.b $xr0, $a0, %pc_lo12(g_i8)
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x32)
+; LA32-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i8x32)
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: copy_i8_to_i8x32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA64-NEXT:    xvldrepl.b $xr0, $a0, %pc_lo12(g_i8)
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x32)
+; LA64-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i8x32)
+; LA64-NEXT:    ret
+;
+; LA64-LARGE-LABEL: copy_i8_to_i8x32:
+; LA64-LARGE:       # %bb.0: # %entry
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    xvldrepl.b $xr0, $a0, 0
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x32)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8x32)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8x32)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8x32)
+; LA64-LARGE-NEXT:    xvstx $xr0, $a1, $a0
+; LA64-LARGE-NEXT:    ret
+entry:
+  %0 = call <32 x i8> @llvm.loongarch.lasx.xvldrepl.b(ptr @g_i8, i32 0)
+  store <32 x i8> %0, ptr @g_i8x32, align 32
+  ret void
+}
+
 @g_rmw = dso_local global i64 0
 
 define dso_local void @rmw() nounwind {
@@ -659,11 +812,11 @@ define dso_local void @control_flow_with_mem_access() nounwind {
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_a32+4)
 ; LA32-NEXT:    ld.w $a1, $a0, %pc_lo12(g_a32+4)
 ; LA32-NEXT:    ori $a2, $zero, 1
-; LA32-NEXT:    blt $a1, $a2, .LBB21_2
+; LA32-NEXT:    blt $a1, $a2, .LBB25_2
 ; LA32-NEXT:  # %bb.1: # %if.then
 ; LA32-NEXT:    ori $a1, $zero, 10
 ; LA32-NEXT:    st.w $a1, $a0, %pc_lo12(g_a32+4)
-; LA32-NEXT:  .LBB21_2: # %if.end
+; LA32-NEXT:  .LBB25_2: # %if.end
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: control_flow_with_mem_access:
@@ -671,11 +824,11 @@ define dso_local void @control_flow_with_mem_access() nounwind {
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_a32+4)
 ; LA64-NEXT:    ld.w $a1, $a0, %pc_lo12(g_a32+4)
 ; LA64-NEXT:    ori $a2, $zero, 1
-; LA64-NEXT:    blt $a1, $a2, .LBB21_2
+; LA64-NEXT:    blt $a1, $a2, .LBB25_2
 ; LA64-NEXT:  # %bb.1: # %if.then
 ; LA64-NEXT:    ori $a1, $zero, 10
 ; LA64-NEXT:    st.w $a1, $a0, %pc_lo12(g_a32+4)
-; LA64-NEXT:  .LBB21_2: # %if.end
+; LA64-NEXT:  .LBB25_2: # %if.end
 ; LA64-NEXT:    ret
 ;
 ; LA64-LARGE-LABEL: control_flow_with_mem_access:
@@ -686,7 +839,7 @@ define dso_local void @control_flow_with_mem_access() nounwind {
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_a32+4)
 ; LA64-LARGE-NEXT:    ldx.w $a0, $a1, $a0
 ; LA64-LARGE-NEXT:    ori $a1, $zero, 1
-; LA64-LARGE-NEXT:    blt $a0, $a1, .LBB21_2
+; LA64-LARGE-NEXT:    blt $a0, $a1, .LBB25_2
 ; LA64-LARGE-NEXT:  # %bb.1: # %if.then
 ; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_a32+4)
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_a32+4)
@@ -694,7 +847,7 @@ define dso_local void @control_flow_with_mem_access() nounwind {
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_a32+4)
 ; LA64-LARGE-NEXT:    ori $a2, $zero, 10
 ; LA64-LARGE-NEXT:    stx.w $a2, $a1, $a0
-; LA64-LARGE-NEXT:  .LBB21_2: # %if.end
+; LA64-LARGE-NEXT:  .LBB25_2: # %if.end
 ; LA64-LARGE-NEXT:    ret
 entry:
   %0 = load i32, ptr getelementptr inbounds ([1 x i32], ptr @g_a32, i32 1), align 4
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll
index bd762a0e1d741f0..ce664c78e86c2ae 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll
@@ -21,8 +21,8 @@ define void @b16() {
 
   ; 32R1:           sll   $[[TMP1:[0-9]+]], $[[A_VAL]], 8
   ; 32R1:           srl   $[[TMP2:[0-9]+]], $[[A_VAL]], 8
-  ; 32R1:           or    $[[TMP3:[0-9]+]], $[[TMP1]], $[[TMP2]]
-  ; 32R1:           andi  $[[TMP4:[0-9]+]], $[[TMP3]], 65535
+  ; 32R1:           andi  $[[TMP3:[0-9]+]], $[[TMP2]], 255
+  ; 32R1:           or    $[[RESULT:[0-9]+]], $[[TMP1]], $[[TMP3]]
 
   ; 32R2:           wsbh  $[[RESULT:[0-9]+]], $[[A_VAL]]
 
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 4cbab8bf24166c5..f029e5f322968b7 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -1,344 +1,1455 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM70 %s
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM80 %s
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck --check-prefixes=CHECK,SM90 %s
 ; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | %ptxas-verify -arch=sm_80 %}
 ; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
 
+target triple = "nvptx64-nvidia-cuda"
+
 ; LDST: .b8 bfloat_array[8] = {1, 2, 3, 4, 5, 6, 7, 8};
 @"bfloat_array" = addrspace(1) constant [4 x bfloat]
                 [bfloat 0xR0201, bfloat 0xR0403, bfloat 0xR0605, bfloat 0xR0807]
 
-; CHECK-LABEL: test_fadd(
-; CHECK-DAG:  ld.param.b16    [[A:%rs[0-9]+]], [test_fadd_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%rs[0-9]+]], [test_fadd_param_1];
-; SM90:       add.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-;
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80-DAG:   cvt.f32.bf16    [[FB:%f[0-9]+]], [[B]];
-; SM80:       add.rn.f32      [[FR:%f[0-9]+]], [[FA]], [[FB]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define bfloat @test_fadd(bfloat %0, bfloat %1) {
+; SM70-LABEL: test_fadd(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fadd_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_fadd_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fadd(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .f32 %f<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fadd_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_fadd_param_1];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs1;
+; SM80-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f3;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fadd(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fadd_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_fadd_param_1];
+; SM90-NEXT:    add.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %3 = fadd bfloat %0, %1
   ret bfloat %3
 }
 
-; CHECK-LABEL: test_fsub(
-; CHECK-DAG:  ld.param.b16    [[A:%rs[0-9]+]], [test_fsub_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%rs[0-9]+]], [test_fsub_param_1];
-; SM90:       sub.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-;
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80-DAG:   cvt.f32.bf16    [[FB:%f[0-9]+]], [[B]];
-; SM80:       sub.rn.f32      [[FR:%f[0-9]+]], [[FA]], [[FB]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define bfloat @test_fsub(bfloat %0, bfloat %1) {
+; SM70-LABEL: test_fsub(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fsub_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_fsub_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fsub(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .f32 %f<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fsub_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_fsub_param_1];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs1;
+; SM80-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f3;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fsub(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fsub_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_fsub_param_1];
+; SM90-NEXT:    sub.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %3 = fsub bfloat %0, %1
   ret bfloat %3
 }
 
-; CHECK-LABEL: test_faddx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_faddx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_faddx2_param_1];
-; SM90:       add.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   add.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   add.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_faddx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    add.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_faddx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    add.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_faddx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_faddx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_faddx2_param_0];
+; SM90-NEXT:    add.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fadd <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fsubx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fsubx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fsubx2_param_1];
-; SM90:       sub.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   sub.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   sub.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fsubx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    sub.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fsubx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    sub.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fsubx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_0];
+; SM90-NEXT:    sub.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fsub <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fmulx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fmulx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fmulx2_param_1];
-; SM90:       mul.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   mul.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   mul.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fmulx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    mul.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    mul.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fmulx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    mul.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    mul.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fmulx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_0];
+; SM90-NEXT:    mul.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fmul <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fdiv(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fdiv_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fdiv_param_1];
-; CHECK-DAG:  mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]]
-; CHECK-DAG:  mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]]
-; CHECK-DAG:  cvt.f32.bf16     [[FA0:%f[0-9]+]], [[A0]];
-; CHECK-DAG:  cvt.f32.bf16     [[FA1:%f[0-9]+]], [[A1]];
-; CHECK-DAG:  cvt.f32.bf16     [[FB0:%f[0-9]+]], [[B0]];
-; CHECK-DAG:  cvt.f32.bf16     [[FB1:%f[0-9]+]], [[B1]];
-; CHECK-DAG:  div.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; CHECK-DAG:  div.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; CHECK-DAG:  cvt.rn.bf16.f32  [[R0:%rs[0-9]+]], [[FR0]];
-; CHECK-DAG:  cvt.rn.bf16.f32  [[R1:%rs[0-9]+]], [[FR1]];
-; CHECK-NEXT: mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]}
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fdiv(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fdiv(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fdiv(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<7>;
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-NEXT:    .reg .f32 %f<7>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM90-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM90-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM90-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM90-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM90-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM90-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fdiv <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_extract_0(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_extract_0_param_0];
-; CHECK:      st.param.b16    [func_retval0+0], [[A]];
-; CHECK:      ret;
-
 define bfloat @test_extract_0(<2 x bfloat> %a) #0 {
+; CHECK-LABEL: test_extract_0(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_extract_0_param_0];
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; CHECK-NEXT:    ret;
   %e = extractelement <2 x bfloat> %a, i32 0
   ret bfloat %e
 }
 
-; CHECK-LABEL: test_extract_1(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_extract_1_param_0+2];
-; CHECK:      st.param.b16    [func_retval0+0], [[A]];
-; CHECK:      ret;
-
 define bfloat @test_extract_1(<2 x bfloat> %a) #0 {
+; CHECK-LABEL: test_extract_1(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_extract_1_param_0+2];
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; CHECK-NEXT:    ret;
   %e = extractelement <2 x bfloat> %a, i32 1
   ret bfloat %e
 }
 
-; CHECK-LABEL: test_fpext_float(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_fpext_float_param_0];
-; CHECK:      cvt.f32.bf16     [[R:%f[0-9]+]], [[A]];
-; CHECK:      st.param.f32    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define float @test_fpext_float(bfloat %a) #0 {
+; SM70-LABEL: test_fpext_float(
+; SM70:       {
+; SM70-NEXT:    .reg .b32 %r<3>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fpext_float_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fpext_float(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fpext_float_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fpext_float(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .f32 %f<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fpext_float_param_0];
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM90-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM90-NEXT:    ret;
   %r = fpext bfloat %a to float
   ret float %r
 }
 
-; CHECK-LABEL: test_fptrunc_float(
-; CHECK:      ld.param.f32    [[A:%f[0-9]+]], [test_fptrunc_float_param_0];
-; CHECK:      cvt.rn.bf16.f32  [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_fptrunc_float(float %a) #0 {
+; SM70-LABEL: test_fptrunc_float(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptrunc_float(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptrunc_float(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .f32 %f<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = fptrunc float %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_fadd_imm_1(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_fadd_imm_1_param_0];
-; SM90:       mov.b16         [[B:%rs[0-9]+]], 0x3F80;
-; SM90:       add.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80:       add.rn.f32      [[FR:%f[0-9]+]], [[FA]], 0f3F800000;
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
 define bfloat @test_fadd_imm_1(bfloat %a) #0 {
+; SM70-LABEL: test_fadd_imm_1(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<9>;
+; SM70-NEXT:    .reg .f32 %f<3>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fadd_imm_1_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    add.rn.f32 %f2, %f1, 0f3F800000;
+; SM70-NEXT:    mov.b32 %r3, %f2;
+; SM70-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; SM70-NEXT:    add.s32 %r5, %r4, %r3;
+; SM70-NEXT:    add.s32 %r6, %r5, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f2, %f2;
+; SM70-NEXT:    or.b32 %r7, %r3, 4194304;
+; SM70-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fadd_imm_1(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<3>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fadd_imm_1_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    add.rn.f32 %f2, %f1, 0f3F800000;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fadd_imm_1(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fadd_imm_1_param_0];
+; SM90-NEXT:    mov.b16 %rs2, 0x3F80;
+; SM90-NEXT:    add.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %r = fadd bfloat %a, 1.0
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_select_cc_bf16_f64(
-; CHECK-DAG:      ld.param.f64    [[A:%fd[0-9]+]], [test_select_cc_bf16_f64_param_0];
-; CHECK-DAG:      ld.param.f64    [[B:%fd[0-9]+]], [test_select_cc_bf16_f64_param_1];
-; CHECK:          setp.lt.f64     [[P:%p[0-9]+]], [[A]], [[B]];
-; CHECK-DAG:      ld.param.b16    [[C:%rs[0-9]+]], [test_select_cc_bf16_f64_param_2];
-; CHECK-DAG:      ld.param.b16    [[D:%rs[0-9]+]], [test_select_cc_bf16_f64_param_3];
-; CHECK:          selp.b16        [[R:%rs[0-9]+]], [[C]], [[D]], [[P]];
-; CHECK-NEXT:     st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT:     ret;
 define bfloat @test_select_cc_bf16_f64(double %a, double %b, bfloat %c, bfloat %d) #0 {
+; CHECK-LABEL: test_select_cc_bf16_f64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .f64 %fd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f64 %fd1, [test_select_cc_bf16_f64_param_0];
+; CHECK-NEXT:    ld.param.f64 %fd2, [test_select_cc_bf16_f64_param_1];
+; CHECK-NEXT:    setp.lt.f64 %p1, %fd1, %fd2;
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_select_cc_bf16_f64_param_2];
+; CHECK-NEXT:    ld.param.b16 %rs2, [test_select_cc_bf16_f64_param_3];
+; CHECK-NEXT:    selp.b16 %rs3, %rs1, %rs2, %p1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; CHECK-NEXT:    ret;
   %cc = fcmp olt double %a, %b
   %r = select i1 %cc, bfloat %c, bfloat %d
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_extload_bf16x8
-; CHECK: ld.shared.v4.b32 {%r
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
 define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
+; SM70-LABEL: test_extload_bf16x8(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<17>;
+; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .f32 %f<9>;
+; SM70-NEXT:    .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM70-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs8;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f1, %r6;
+; SM70-NEXT:    cvt.u32.u16 %r7, %rs7;
+; SM70-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NEXT:    mov.b32 %f2, %r8;
+; SM70-NEXT:    cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT:    shl.b32 %r10, %r9, 16;
+; SM70-NEXT:    mov.b32 %f3, %r10;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs5;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
+; SM70-NEXT:    mov.b32 %f4, %r12;
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs4;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f5, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f6, %r16;
+; SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; SM70-NEXT:    mov.b32 %f7, %r18;
+; SM70-NEXT:    cvt.u32.u16 %r19, %rs1;
+; SM70-NEXT:    shl.b32 %r20, %r19, 16;
+; SM70-NEXT:    mov.b32 %f8, %r20;
+; SM70-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM70-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_extload_bf16x8(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<9>;
+; SM80-NEXT:    .reg .b32 %r<5>;
+; SM80-NEXT:    .reg .f32 %f<9>;
+; SM80-NEXT:    .reg .b64 %rd<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM80-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs8;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs7;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs6;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs5;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f6, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f7, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f8, %rs1;
+; SM80-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM80-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_extload_bf16x8(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<9>;
+; SM90-NEXT:    .reg .b32 %r<5>;
+; SM90-NEXT:    .reg .f32 %f<9>;
+; SM90-NEXT:    .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM90-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs8;
+; SM90-NEXT:    cvt.f32.bf16 %f2, %rs7;
+; SM90-NEXT:    cvt.f32.bf16 %f3, %rs6;
+; SM90-NEXT:    cvt.f32.bf16 %f4, %rs5;
+; SM90-NEXT:    cvt.f32.bf16 %f5, %rs4;
+; SM90-NEXT:    cvt.f32.bf16 %f6, %rs3;
+; SM90-NEXT:    cvt.f32.bf16 %f7, %rs2;
+; SM90-NEXT:    cvt.f32.bf16 %f8, %rs1;
+; SM90-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM90-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM90-NEXT:    ret;
   %load = load <8 x bfloat>, ptr addrspace(3) %arg, align 16
   %res = fpext <8 x bfloat> %load to <8 x float>
   ret <8 x float> %res
 }
 
-; CHECK-LABEL: test_fptosi_i16(
-; CHECK:      ld.param.b16     [[A:%rs[0-9]+]], [test_fptosi_i16_param_0];
-; SM80:       cvt.f32.bf16     [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rzi.s16.f32  [[C:%rs[0-9]+]], [[B]];
-; SM80:       cvt.u32.u16      [[R:%r[0-9]+]], [[C]];
-; SM90:       cvt.rzi.s16.bf16 [[B:%rs[0-9]+]], [[A]];
-; SM90:       cvt.u32.u16      [[R:%r[0-9]+]], [[B]];
-; CHECK:      st.param.b32     [func_retval0+0], [[R]];
-; CHECK:      ret;
 define i16 @test_fptosi_i16(bfloat %a) {
+; SM70-LABEL: test_fptosi_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<4>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fptosi_i16_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rzi.s16.f32 %rs1, %f1;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptosi_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fptosi_i16_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rzi.s16.f32 %rs2, %f1;
+; SM80-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptosi_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fptosi_i16_param_0];
+; SM90-NEXT:    cvt.rzi.s16.bf16 %rs2, %rs1;
+; SM90-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM90-NEXT:    ret;
   %r = fptosi bfloat %a to i16
   ret i16 %r
 }
 
-; CHECK-LABEL: test_fptoui_i16(
-; CHECK:      ld.param.b16     [[A:%rs[0-9]+]], [test_fptoui_i16_param_0];
-; SM80:       cvt.f32.bf16     [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rzi.u16.f32  [[C:%rs[0-9]+]], [[B]];
-; SM80:       cvt.u32.u16      [[R:%r[0-9]+]], [[C]];
-; SM90:       cvt.rzi.u16.bf16 [[B:%rs[0-9]+]], [[A]];
-; SM90:       cvt.u32.u16      [[R:%r[0-9]+]], [[B]];
-; CHECK:      st.param.b32     [func_retval0+0], [[R]];
-; CHECK:      ret;
 define i16 @test_fptoui_i16(bfloat %a) {
+; SM70-LABEL: test_fptoui_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<4>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fptoui_i16_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rzi.u16.f32 %rs1, %f1;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptoui_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fptoui_i16_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rzi.u16.f32 %rs2, %f1;
+; SM80-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptoui_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fptoui_i16_param_0];
+; SM90-NEXT:    cvt.rzi.u16.bf16 %rs2, %rs1;
+; SM90-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM90-NEXT:    ret;
   %r = fptoui bfloat %a to i16
   ret i16 %r
 }
 
-; CHECK-LABEL: test_sitofp_i16(
-; CHECK:      ld.param.u16    [[A:%rs[0-9]+]], [test_sitofp_i16_param_0];
-; SM80:       cvt.rn.f32.s16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.s16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_sitofp_i16(i16 %a) {
+; SM70-LABEL: test_sitofp_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM70-NEXT:    cvt.rn.f32.s16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_sitofp_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM80-NEXT:    cvt.rn.f32.s16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_sitofp_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM90-NEXT:    cvt.rn.bf16.s16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = sitofp i16 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i8(
-; CHECK:      ld.param.u8 %rs1, [test_uitofp_i8_param_0];
-; SM80:       cvt.rn.f32.u16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i8(i8 %a) {
+; SM70-LABEL: test_uitofp_i8(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM70-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i8(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM80-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i8(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = uitofp i8 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i1(
-; CHECK:      ld.param.u8     [[A:%rs[0-9]+]], [test_uitofp_i1_param_0];
-; CHECK:      and.b16         [[B:%rs[0-9]+]], [[A]], 1;
-; CHECK:      setp.eq.b16     [[C:%p[0-9]+]], [[B]], 1;
-; CHECK:      selp.u32        [[D:%r[0-9]+]], 1, 0, [[C]];
-; SM80:       cvt.rn.f32.u32  [[E:%f[0-9]+]], [[D]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[E]];
-; SM90:       cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[D]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i1(i1 %a) {
+; SM70-LABEL: test_uitofp_i1(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<5>;
+; SM70-NEXT:    .reg .b32 %r<8>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM70-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM70-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM70-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM70-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM70-NEXT:    mov.b32 %r2, %f1;
+; SM70-NEXT:    bfe.u32 %r3, %r2, 16, 1;
+; SM70-NEXT:    add.s32 %r4, %r3, %r2;
+; SM70-NEXT:    add.s32 %r5, %r4, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f1, %f1;
+; SM70-NEXT:    or.b32 %r6, %r2, 4194304;
+; SM70-NEXT:    selp.b32 %r7, %r6, %r5, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs3}, %r7; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i1(
+; SM80:       {
+; SM80-NEXT:    .reg .pred %p<2>;
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM80-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM80-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM80-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM80-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i1(
+; SM90:       {
+; SM90-NEXT:    .reg .pred %p<2>;
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM90-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM90-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM90-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM90-NEXT:    cvt.rn.bf16.u32 %rs3, %r1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %r = uitofp i1 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i16(
-; CHECK:      ld.param.u16    [[A:%rs[0-9]+]], [test_uitofp_i16_param_0];
-; SM80:       cvt.rn.f32.u16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i16(i16 %a) {
+; SM70-LABEL: test_uitofp_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM70-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM80-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = uitofp i16 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i32(
-; CHECK:      ld.param.u32    [[A:%r[0-9]+]], [test_uitofp_i32_param_0];
-; SM80:       cvt.rn.f32.u32  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i32(i32 %a) {
+; SM70-LABEL: test_uitofp_i32(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<8>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM70-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM70-NEXT:    mov.b32 %r2, %f1;
+; SM70-NEXT:    bfe.u32 %r3, %r2, 16, 1;
+; SM70-NEXT:    add.s32 %r4, %r3, %r2;
+; SM70-NEXT:    add.s32 %r5, %r4, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r6, %r2, 4194304;
+; SM70-NEXT:    selp.b32 %r7, %r6, %r5, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i32(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM80-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i32(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u32 %rs1, %r1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = uitofp i32 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i64(
-; CHECK:      ld.param.u64    [[A:%rd[0-9]+]], [test_uitofp_i64_param_0];
-; SM80:       cvt.rn.f32.u64  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u64 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i64(i64 %a) {
+; SM70-LABEL: test_uitofp_i64(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-NEXT:    .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM70-NEXT:    cvt.rn.f32.u64 %f1, %rd1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i64(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-NEXT:    .reg .b64 %rd<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM80-NEXT:    cvt.rn.f32.u64 %f1, %rd1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i64(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u64 %rs1, %rd1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = uitofp i64 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_roundeven(
-; CHECK:      ld.param.b16      [[A:%rs[0-9]+]], [test_roundeven_param_0];
-; SM80:       cvt.rni.f32.f32   [[F:%f[0-9]+]]
-; SM80:       cvt.rn.bf16.f32   [[R:%rs[0-9]+]], [[F]];
-; SM90:       cvt.rni.bf16.bf16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16      [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_roundeven(bfloat %a) {
+; SM70-LABEL: test_roundeven(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<9>;
+; SM70-NEXT:    .reg .f32 %f<3>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_roundeven_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rni.f32.f32 %f2, %f1;
+; SM70-NEXT:    mov.b32 %r3, %f2;
+; SM70-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; SM70-NEXT:    add.s32 %r5, %r4, %r3;
+; SM70-NEXT:    add.s32 %r6, %r5, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f2, %f2;
+; SM70-NEXT:    or.b32 %r7, %r3, 4194304;
+; SM70-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_roundeven(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<3>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_roundeven_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rni.f32.f32 %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_roundeven(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_roundeven_param_0];
+; SM90-NEXT:    cvt.rni.bf16.bf16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
   ret bfloat %r
 }
+
+define bfloat @test_maximum(bfloat %a, bfloat %b) {
+; SM70-LABEL: test_maximum(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<6>;
+; SM70-NEXT:    .reg .b16 %rs<11>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM70-NEXT:    ld.param.b16 %rs3, [test_maximum_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs3;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    setp.gt.f32 %p1, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs5, %rs1, %rs3, %p1;
+; SM70-NEXT:    setp.nan.f32 %p2, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT:    setp.eq.s16 %p3, %rs1, 0;
+; SM70-NEXT:    selp.b16 %rs7, %rs1, %rs6, %p3;
+; SM70-NEXT:    setp.eq.s16 %p4, %rs3, 0;
+; SM70-NEXT:    selp.b16 %rs8, %rs3, %rs7, %p4;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs6;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f3, %r6;
+; SM70-NEXT:    setp.eq.f32 %p5, %f3, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs10;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maximum(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_maximum_param_1];
+; SM80-NEXT:    max.NaN.bf16 %rs3, %rs1, %rs2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maximum(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_maximum_param_1];
+; SM90-NEXT:    max.NaN.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
+  %r = call bfloat @llvm.maximum.bf16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_maxnum(bfloat %a, bfloat %b) {
+; SM70-LABEL: test_maxnum(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_maxnum_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_maxnum_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    max.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maxnum(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_maxnum_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_maxnum_param_1];
+; SM80-NEXT:    max.bf16 %rs3, %rs1, %rs2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maxnum(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_maxnum_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_maxnum_param_1];
+; SM90-NEXT:    max.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
+  %r = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; SM70-LABEL: test_maximum_v2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<11>;
+; SM70-NEXT:    .reg .b16 %rs<21>;
+; SM70-NEXT:    .reg .b32 %r<16>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    setp.gt.f32 %p1, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs7, %rs5, %rs2, %p1;
+; SM70-NEXT:    setp.nan.f32 %p2, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs8, 0x7FC0, %rs7, %p2;
+; SM70-NEXT:    setp.eq.s16 %p3, %rs5, 0;
+; SM70-NEXT:    selp.b16 %rs9, %rs5, %rs8, %p3;
+; SM70-NEXT:    setp.eq.s16 %p4, %rs2, 0;
+; SM70-NEXT:    selp.b16 %rs10, %rs2, %rs9, %p4;
+; SM70-NEXT:    cvt.u32.u16 %r7, %rs8;
+; SM70-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NEXT:    mov.b32 %f3, %r8;
+; SM70-NEXT:    setp.eq.f32 %p5, %f3, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs12, %rs10, %rs8, %p5;
+; SM70-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT:    shl.b32 %r10, %r9, 16;
+; SM70-NEXT:    mov.b32 %f4, %r10;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs4;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
+; SM70-NEXT:    mov.b32 %f5, %r12;
+; SM70-NEXT:    setp.gt.f32 %p6, %f5, %f4;
+; SM70-NEXT:    selp.b16 %rs15, %rs4, %rs1, %p6;
+; SM70-NEXT:    setp.nan.f32 %p7, %f5, %f4;
+; SM70-NEXT:    selp.b16 %rs16, 0x7FC0, %rs15, %p7;
+; SM70-NEXT:    setp.eq.s16 %p8, %rs4, 0;
+; SM70-NEXT:    selp.b16 %rs17, %rs4, %rs16, %p8;
+; SM70-NEXT:    setp.eq.s16 %p9, %rs1, 0;
+; SM70-NEXT:    selp.b16 %rs18, %rs1, %rs17, %p9;
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs16;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f6, %r14;
+; SM70-NEXT:    setp.eq.f32 %p10, %f6, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs20, %rs18, %rs16, %p10;
+; SM70-NEXT:    mov.b32 %r15, {%rs20, %rs12};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r15;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maximum_v2(
+; SM80:       {
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
+; SM80-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
+; SM80-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maximum_v2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
+; SM90-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
+  %r = call <2 x bfloat> @llvm.maximum.bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; SM70-LABEL: test_maxnum_v2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    max.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    max.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maxnum_v2(
+; SM80:       {
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
+; SM80-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
+; SM80-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maxnum_v2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
+; SM90-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
+  %r = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+  ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/brkpt.ll b/llvm/test/CodeGen/NVPTX/brkpt.ll
new file mode 100644
index 000000000000000..beb2ff6aed6c9c4
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/brkpt.ll
@@ -0,0 +1,8 @@
+; RUN: llc -o - -march=nvptx64 %s | FileCheck %s
+
+; CHECK-LABEL: .func breakpoint
+define void @breakpoint() {
+  ; CHECK: brkpt;
+  call void @llvm.debugtrap()
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/convert-sm89.ll b/llvm/test/CodeGen/NVPTX/convert-sm89.ll
new file mode 100644
index 000000000000000..5a1a640afc1cf9c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/convert-sm89.ll
@@ -0,0 +1,86 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_89 -mattr=+ptx81 | FileCheck %s
+; RUN: %if ptxas-12.1 %{ llc < %s -march=nvptx64 -mcpu=sm_89 -mattr=+ptx81 | %ptxas-verify -arch=sm_89 %}
+
+; CHECK-LABEL: cvt_rn_e4m3x2_f32
+define i16 @cvt_rn_e4m3x2_f32(float %f1, float %f2) {
+; CHECK: cvt.rn.satfinite.e4m3x2.f32
+  %val = call i16 @llvm.nvvm.ff.to.e4m3x2.rn(float %f1, float %f2);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_e4m3x2_f32
+define i16 @cvt_rn_relu_e4m3x2_f32(float %f1, float %f2) {
+; CHECK: cvt.rn.satfinite.relu.e4m3x2.f32
+  %val = call i16 @llvm.nvvm.ff.to.e4m3x2.rn.relu(float %f1, float %f2);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_e5m2x2_f32
+define i16 @cvt_rn_e5m2x2_f32(float %f1, float %f2) {
+; CHECK: cvt.rn.satfinite.e5m2x2.f32
+  %val = call i16 @llvm.nvvm.ff.to.e5m2x2.rn(float %f1, float %f2);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_e5m2x2_f32
+define i16 @cvt_rn_relu_e5m2x2_f32(float %f1, float %f2) {
+; CHECK: cvt.rn.satfinite.relu.e5m2x2.f32
+  %val = call i16 @llvm.nvvm.ff.to.e5m2x2.rn.relu(float %f1, float %f2);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_e4m3x2_f16x2
+define i16 @cvt_rn_e4m3x2_f16x2(<2 x half> %in) {
+; CHECK: cvt.rn.satfinite.e4m3x2.f16x2
+  %val = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn(<2 x half> %in);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_e4m3x2_f16x2
+define i16 @cvt_rn_relu_e4m3x2_f16x2(<2 x half> %in) {
+; CHECK: cvt.rn.satfinite.relu.e4m3x2.f16x2
+  %val = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn.relu(<2 x half> %in);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_e5m2x2_f16x2
+define i16 @cvt_rn_e5m2x2_f16x2(<2 x half> %in) {
+; CHECK: cvt.rn.satfinite.e5m2x2.f16x2
+  %val = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn(<2 x half> %in);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_e5m2x2_f16x2
+define i16 @cvt_rn_relu_e5m2x2_f16x2(<2 x half> %in) {
+; CHECK: cvt.rn.satfinite.relu.e5m2x2.f16x2
+  %val = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn.relu(<2 x half> %in);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_f16x2_e4m3x2
+define <2 x half> @cvt_rn_f16x2_e4m3x2(i16 %in) {
+; CHECK: cvt.rn.f16x2.e4m3x2
+  %val = call <2 x half> @llvm.nvvm.e4m3x2.to.f16x2.rn(i16 %in);
+  ret <2 x half> %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_f16x2_e4m3x2
+define <2 x half> @cvt_rn_relu_f16x2_e4m3x2(i16 %in) {
+; CHECK: cvt.rn.relu.f16x2.e4m3x2
+  %val = call <2 x half> @llvm.nvvm.e4m3x2.to.f16x2.rn.relu(i16 %in);
+  ret <2 x half> %val
+}
+
+; CHECK-LABEL: cvt_rn_f16x2_e5m2x2
+define <2 x half> @cvt_rn_f16x2_e5m2x2(i16 %in) {
+; CHECK: cvt.rn.f16x2.e5m2x2
+  %val = call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn(i16 %in);
+  ret <2 x half> %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_f16x2_e5m2x2
+define <2 x half> @cvt_rn_relu_f16x2_e5m2x2(i16 %in) {
+; CHECK: cvt.rn.relu.f16x2.e5m2x2
+  %val = call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn.relu(i16 %in);
+  ret <2 x half> %val
+}
diff --git a/llvm/test/CodeGen/PowerPC/merge-private.ll b/llvm/test/CodeGen/PowerPC/merge-private.ll
index 6ed2d6dfc542b76..0ca706abb275fcc 100644
--- a/llvm/test/CodeGen/PowerPC/merge-private.ll
+++ b/llvm/test/CodeGen/PowerPC/merge-private.ll
@@ -11,6 +11,12 @@
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-unknown-linux -mcpu=pwr8 \
 ; RUN:     -ppc-asm-full-reg-names -ppc-global-merge=true < %s | FileCheck %s \
 ; RUN:     --check-prefix=LINUX64BE
+; The below run line is added to ensure that the assembly corresponding to
+; the following check-prefix is generated by default on AIX (without any
+; options).
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr8 \
+; RUN:     -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:     --check-prefix=AIX64
 
 @.str = private unnamed_addr constant [15 x i8] c"Private global\00", align 1
 @str = internal constant [16 x i8] c"Internal global\00", align 1
diff --git a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll
new file mode 100644
index 000000000000000..fde135246c2a98c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll
@@ -0,0 +1,373 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:     -ppc-asm-full-reg-names -enable-global-merge=false < %s | FileCheck %s \
+; RUN:     --check-prefix=CHECK64
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -ppc-asm-full-reg-names -enable-global-merge=false < %s | FileCheck %s \
+; RUN:     --check-prefix=CHECK32
+; RUN: llc -verify-machineinstrs -mtriple powerpc64le-unknown-linux \
+; RUN:     -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:     --check-prefix=LINUX64LE
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-unknown-linux \
+; RUN:     -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:     --check-prefix=LINUX64BE
+
+@.str = private unnamed_addr constant [47 x i8] c"TLS variable 1, 2 and non-TLS var: %s, %s, %s\0A\00", align 1
+@a = internal thread_local constant [5 x i8] c"tls1\00", align 1
+@b = internal thread_local constant [5 x i8] c"tls2\00", align 1
+@c = internal constant [15 x i8] c"Regular global\00", align 1
+@d = internal constant [10 x i32] [i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10], align 4
+@e = internal constant [4 x float] [float 0x4055F33340000000, float 0x4056333340000000, float 0x40567999A0000000, float 0x4056B33340000000], align 4
+
+declare noundef signext i32 @printf(ptr nocapture noundef readonly, ...) local_unnamed_addr #0
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
+declare void @callee(ptr noundef) local_unnamed_addr #3
+declare void @callee2(ptr noundef) local_unnamed_addr #3
+
+define void @print_tls_func() {
+; CHECK64-LABEL: print_tls_func:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    mflr r0
+; CHECK64-NEXT:    stdu r1, -112(r1)
+; CHECK64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; CHECK64-NEXT:    std r0, 128(r1)
+; CHECK64-NEXT:    ld r6, L..C1(r2) # @__ModuleStringPool
+; CHECK64-NEXT:    bla .__tls_get_mod[PR]
+; CHECK64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsld) @a
+; CHECK64-NEXT:    ld r5, L..C3(r2) # target-flags(ppc-tlsld) @b
+; CHECK64-NEXT:    add r4, r3, r4
+; CHECK64-NEXT:    add r5, r3, r5
+; CHECK64-NEXT:    addi r3, r6, 71
+; CHECK64-NEXT:    addi r6, r6, 56
+; CHECK64-NEXT:    bl .printf[PR]
+; CHECK64-NEXT:    nop
+; CHECK64-NEXT:    addi r1, r1, 112
+; CHECK64-NEXT:    ld r0, 16(r1)
+; CHECK64-NEXT:    mtlr r0
+; CHECK64-NEXT:    blr
+;
+; CHECK32-LABEL: print_tls_func:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    mflr r0
+; CHECK32-NEXT:    stwu r1, -64(r1)
+; CHECK32-NEXT:    lwz r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; CHECK32-NEXT:    stw r0, 72(r1)
+; CHECK32-NEXT:    lwz r6, L..C1(r2) # @__ModuleStringPool
+; CHECK32-NEXT:    bla .__tls_get_mod[PR]
+; CHECK32-NEXT:    lwz r4, L..C2(r2) # target-flags(ppc-tlsld) @a
+; CHECK32-NEXT:    lwz r5, L..C3(r2) # target-flags(ppc-tlsld) @b
+; CHECK32-NEXT:    add r4, r3, r4
+; CHECK32-NEXT:    add r5, r3, r5
+; CHECK32-NEXT:    addi r3, r6, 71
+; CHECK32-NEXT:    addi r6, r6, 56
+; CHECK32-NEXT:    bl .printf[PR]
+; CHECK32-NEXT:    nop
+; CHECK32-NEXT:    addi r1, r1, 64
+; CHECK32-NEXT:    lwz r0, 8(r1)
+; CHECK32-NEXT:    mtlr r0
+; CHECK32-NEXT:    blr
+;
+; LINUX64LE-LABEL: print_tls_func:
+; LINUX64LE:       # %bb.0: # %entry
+; LINUX64LE-NEXT:    mflr r0
+; LINUX64LE-NEXT:    stdu r1, -96(r1)
+; LINUX64LE-NEXT:    std r0, 112(r1)
+; LINUX64LE-NEXT:    .cfi_def_cfa_offset 96
+; LINUX64LE-NEXT:    .cfi_offset lr, 16
+; LINUX64LE-NEXT:    addis r3, r13, a@tprel@ha
+; LINUX64LE-NEXT:    addi r4, r3, a@tprel@l
+; LINUX64LE-NEXT:    addis r3, r13, b@tprel@ha
+; LINUX64LE-NEXT:    addi r5, r3, b@tprel@l
+; LINUX64LE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64LE-NEXT:    addi r6, r3, .L__ModuleStringPool@toc@l
+; LINUX64LE-NEXT:    addi r3, r6, 71
+; LINUX64LE-NEXT:    addi r6, r6, 56
+; LINUX64LE-NEXT:    bl printf
+; LINUX64LE-NEXT:    nop
+; LINUX64LE-NEXT:    addi r1, r1, 96
+; LINUX64LE-NEXT:    ld r0, 16(r1)
+; LINUX64LE-NEXT:    mtlr r0
+; LINUX64LE-NEXT:    blr
+;
+; LINUX64BE-LABEL: print_tls_func:
+; LINUX64BE:       # %bb.0: # %entry
+; LINUX64BE-NEXT:    mflr r0
+; LINUX64BE-NEXT:    stdu r1, -128(r1)
+; LINUX64BE-NEXT:    std r0, 144(r1)
+; LINUX64BE-NEXT:    .cfi_def_cfa_offset 128
+; LINUX64BE-NEXT:    .cfi_offset lr, 16
+; LINUX64BE-NEXT:    .cfi_offset r30, -16
+; LINUX64BE-NEXT:    addis r3, r2, a@got@tlsld@ha
+; LINUX64BE-NEXT:    std r30, 112(r1) # 8-byte Folded Spill
+; LINUX64BE-NEXT:    addi r3, r3, a@got@tlsld@l
+; LINUX64BE-NEXT:    bl __tls_get_addr(a@tlsld)
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    addis r4, r2, b@got@tlsld@ha
+; LINUX64BE-NEXT:    addis r3, r3, a@dtprel@ha
+; LINUX64BE-NEXT:    addi r30, r3, a@dtprel@l
+; LINUX64BE-NEXT:    addi r3, r4, b@got@tlsld@l
+; LINUX64BE-NEXT:    bl __tls_get_addr(b@tlsld)
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    addis r3, r3, b@dtprel@ha
+; LINUX64BE-NEXT:    addi r5, r3, b@dtprel@l
+; LINUX64BE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64BE-NEXT:    addi r4, r3, .L__ModuleStringPool@toc@l
+; LINUX64BE-NEXT:    addi r3, r4, 71
+; LINUX64BE-NEXT:    addi r6, r4, 56
+; LINUX64BE-NEXT:    mr r4, r30
+; LINUX64BE-NEXT:    bl printf
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    ld r30, 112(r1) # 8-byte Folded Reload
+; LINUX64BE-NEXT:    addi r1, r1, 128
+; LINUX64BE-NEXT:    ld r0, 16(r1)
+; LINUX64BE-NEXT:    mtlr r0
+; LINUX64BE-NEXT:    blr
+entry:
+  %0 = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @a)
+  %1 = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @b)
+  %call = tail call signext i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str, ptr noundef nonnull %0, ptr noundef nonnull %1, ptr noundef nonnull @c)
+  ret void
+}
+
+define void @test_func() {
+; CHECK64-LABEL: test_func:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    mflr r0
+; CHECK64-NEXT:    stdu r1, -112(r1)
+; CHECK64-NEXT:    ld r3, L..C1(r2) # @__ModuleStringPool
+; CHECK64-NEXT:    std r0, 128(r1)
+; CHECK64-NEXT:    addi r3, r3, 16
+; CHECK64-NEXT:    bl .callee[PR]
+; CHECK64-NEXT:    nop
+; CHECK64-NEXT:    addi r1, r1, 112
+; CHECK64-NEXT:    ld r0, 16(r1)
+; CHECK64-NEXT:    mtlr r0
+; CHECK64-NEXT:    blr
+;
+; CHECK32-LABEL: test_func:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    mflr r0
+; CHECK32-NEXT:    stwu r1, -64(r1)
+; CHECK32-NEXT:    lwz r3, L..C1(r2) # @__ModuleStringPool
+; CHECK32-NEXT:    stw r0, 72(r1)
+; CHECK32-NEXT:    addi r3, r3, 16
+; CHECK32-NEXT:    bl .callee[PR]
+; CHECK32-NEXT:    nop
+; CHECK32-NEXT:    addi r1, r1, 64
+; CHECK32-NEXT:    lwz r0, 8(r1)
+; CHECK32-NEXT:    mtlr r0
+; CHECK32-NEXT:    blr
+;
+; LINUX64LE-LABEL: test_func:
+; LINUX64LE:       # %bb.0: # %entry
+; LINUX64LE-NEXT:    mflr r0
+; LINUX64LE-NEXT:    stdu r1, -32(r1)
+; LINUX64LE-NEXT:    std r0, 48(r1)
+; LINUX64LE-NEXT:    .cfi_def_cfa_offset 32
+; LINUX64LE-NEXT:    .cfi_offset lr, 16
+; LINUX64LE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64LE-NEXT:    addi r3, r3, .L__ModuleStringPool@toc@l
+; LINUX64LE-NEXT:    addi r3, r3, 16
+; LINUX64LE-NEXT:    bl callee
+; LINUX64LE-NEXT:    nop
+; LINUX64LE-NEXT:    addi r1, r1, 32
+; LINUX64LE-NEXT:    ld r0, 16(r1)
+; LINUX64LE-NEXT:    mtlr r0
+; LINUX64LE-NEXT:    blr
+;
+; LINUX64BE-LABEL: test_func:
+; LINUX64BE:       # %bb.0: # %entry
+; LINUX64BE-NEXT:    mflr r0
+; LINUX64BE-NEXT:    stdu r1, -112(r1)
+; LINUX64BE-NEXT:    std r0, 128(r1)
+; LINUX64BE-NEXT:    .cfi_def_cfa_offset 112
+; LINUX64BE-NEXT:    .cfi_offset lr, 16
+; LINUX64BE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64BE-NEXT:    addi r3, r3, .L__ModuleStringPool@toc@l
+; LINUX64BE-NEXT:    addi r3, r3, 16
+; LINUX64BE-NEXT:    bl callee
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    addi r1, r1, 112
+; LINUX64BE-NEXT:    ld r0, 16(r1)
+; LINUX64BE-NEXT:    mtlr r0
+; LINUX64BE-NEXT:    blr
+entry:
+  tail call void @callee(ptr noundef nonnull @d) #4
+  ret void
+}
+
+define void @test_func2() {
+; CHECK64-LABEL: test_func2:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    mflr r0
+; CHECK64-NEXT:    stdu r1, -112(r1)
+; CHECK64-NEXT:    ld r3, L..C1(r2) # @__ModuleStringPool
+; CHECK64-NEXT:    std r0, 128(r1)
+; CHECK64-NEXT:    bl .callee2[PR]
+; CHECK64-NEXT:    nop
+; CHECK64-NEXT:    addi r1, r1, 112
+; CHECK64-NEXT:    ld r0, 16(r1)
+; CHECK64-NEXT:    mtlr r0
+; CHECK64-NEXT:    blr
+;
+; CHECK32-LABEL: test_func2:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    mflr r0
+; CHECK32-NEXT:    stwu r1, -64(r1)
+; CHECK32-NEXT:    lwz r3, L..C1(r2) # @__ModuleStringPool
+; CHECK32-NEXT:    stw r0, 72(r1)
+; CHECK32-NEXT:    bl .callee2[PR]
+; CHECK32-NEXT:    nop
+; CHECK32-NEXT:    addi r1, r1, 64
+; CHECK32-NEXT:    lwz r0, 8(r1)
+; CHECK32-NEXT:    mtlr r0
+; CHECK32-NEXT:    blr
+;
+; LINUX64LE-LABEL: test_func2:
+; LINUX64LE:       # %bb.0: # %entry
+; LINUX64LE-NEXT:    mflr r0
+; LINUX64LE-NEXT:    stdu r1, -32(r1)
+; LINUX64LE-NEXT:    std r0, 48(r1)
+; LINUX64LE-NEXT:    .cfi_def_cfa_offset 32
+; LINUX64LE-NEXT:    .cfi_offset lr, 16
+; LINUX64LE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64LE-NEXT:    addi r3, r3, .L__ModuleStringPool@toc@l
+; LINUX64LE-NEXT:    bl callee2
+; LINUX64LE-NEXT:    nop
+; LINUX64LE-NEXT:    addi r1, r1, 32
+; LINUX64LE-NEXT:    ld r0, 16(r1)
+; LINUX64LE-NEXT:    mtlr r0
+; LINUX64LE-NEXT:    blr
+;
+; LINUX64BE-LABEL: test_func2:
+; LINUX64BE:       # %bb.0: # %entry
+; LINUX64BE-NEXT:    mflr r0
+; LINUX64BE-NEXT:    stdu r1, -112(r1)
+; LINUX64BE-NEXT:    std r0, 128(r1)
+; LINUX64BE-NEXT:    .cfi_def_cfa_offset 112
+; LINUX64BE-NEXT:    .cfi_offset lr, 16
+; LINUX64BE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64BE-NEXT:    addi r3, r3, .L__ModuleStringPool@toc@l
+; LINUX64BE-NEXT:    bl callee2
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    addi r1, r1, 112
+; LINUX64BE-NEXT:    ld r0, 16(r1)
+; LINUX64BE-NEXT:    mtlr r0
+; LINUX64BE-NEXT:    blr
+entry:
+  tail call void @callee2(ptr noundef nonnull @e) #4
+  ret void
+}
+
+; Check the contents of the TLS data and the __ModuleStringPool structure to
+; check that TLS data has been skipped during string pool merging.
+
+; CHECK64: 	.csect a[TL],2
+; CHECK64-NEXT:	.lglobl	a[TL]
+; CHECK64-NEXT:	.string	"tls1"
+; CHECK64:	.csect b[TL],2
+; CHECK64-NEXT:	.lglobl	b[TL]
+; CHECK64-NEXT:	.string	"tls2"
+; CHECK64:	.csect L..__ModuleStringPool[RO],2
+; CHECK64-NEXT:	.align	2
+; CHECK64-NEXT:	.vbyte	4, 0x42af999a
+; CHECK64-NEXT:	.vbyte	4, 0x42b1999a
+; CHECK64-NEXT:	.vbyte	4, 0x42b3cccd
+; CHECK64-NEXT:	.vbyte	4, 0x42b5999a
+; CHECK64-NEXT:	.vbyte	4, 1
+; CHECK64-NEXT:	.vbyte	4, 2
+; CHECK64-NEXT:	.vbyte	4, 3
+; CHECK64-NEXT:	.vbyte	4, 4
+; CHECK64-NEXT:	.vbyte	4, 5
+; CHECK64-NEXT:	.vbyte	4, 6
+; CHECK64-NEXT:	.vbyte	4, 7
+; CHECK64-NEXT:	.vbyte	4, 8
+; CHECK64-NEXT:	.vbyte	4, 9
+; CHECK64-NEXT:	.vbyte	4, 10
+; CHECK64-NEXT:	.string	"Regular global"
+; CHECK64-NEXT:	.byte	'T,'L,'S,' ,'v,'a,'r,'i,'a,'b,'l,'e,' ,'1,',,' ,'2,' ,'a,'n,'d,' ,'n,'o,'n,'-,'T,'L,'S,' ,'v,'a,'r,':,' ,'%,'s,',,' ,'%,'s,',,' ,'%,'s,0012,0000
+; CHECK64: L..C1:
+; CHECK64-NEXT:	.tc L..__ModuleStringPool[TC],L..__ModuleStringPool[RO]
+; CHECK64: L..C2:
+; CHECK64-NEXT:	.tc a[TC],a[TL]@ld
+; CHECK64: L..C3:
+; CHECK64-NEXT:	.tc b[TC],b[TL]@ld
+
+; CHECK32: 	.csect a[TL],2
+; CHECK32-NEXT:	.lglobl	a[TL]
+; CHECK32-NEXT:	.string	"tls1"
+; CHECK32:	.csect b[TL],2
+; CHECK32-NEXT:	.lglobl	b[TL]
+; CHECK32-NEXT:	.string	"tls2"
+; CHECK32:	.csect L..__ModuleStringPool[RO],2
+; CHECK32-NEXT:	.align	2
+; CHECK32-NEXT:	.vbyte	4, 0x42af999a
+; CHECK32-NEXT:	.vbyte	4, 0x42b1999a
+; CHECK32-NEXT:	.vbyte	4, 0x42b3cccd
+; CHECK32-NEXT:	.vbyte	4, 0x42b5999a
+; CHECK32-NEXT:	.vbyte	4, 1
+; CHECK32-NEXT:	.vbyte	4, 2
+; CHECK32-NEXT:	.vbyte	4, 3
+; CHECK32-NEXT:	.vbyte	4, 4
+; CHECK32-NEXT:	.vbyte	4, 5
+; CHECK32-NEXT:	.vbyte	4, 6
+; CHECK32-NEXT:	.vbyte	4, 7
+; CHECK32-NEXT:	.vbyte	4, 8
+; CHECK32-NEXT:	.vbyte	4, 9
+; CHECK32-NEXT:	.vbyte	4, 10
+; CHECK32-NEXT:	.string	"Regular global"
+; CHECK32-NEXT:	.byte	'T,'L,'S,' ,'v,'a,'r,'i,'a,'b,'l,'e,' ,'1,',,' ,'2,' ,'a,'n,'d,' ,'n,'o,'n,'-,'T,'L,'S,' ,'v,'a,'r,':,' ,'%,'s,',,' ,'%,'s,',,' ,'%,'s,0012,0000
+; CHECK32: L..C1:
+; CHECK32-NEXT:	.tc L..__ModuleStringPool[TC],L..__ModuleStringPool[RO]
+; CHECK32: L..C2:
+; CHECK32-NEXT:	.tc a[TC],a[TL]@ld
+; CHECK32: L..C3:
+; CHECK32-NEXT:	.tc b[TC],b[TL]@ld
+
+; LINUX64LE: a:
+; LINUX64LE-NEXT:	.asciz	"tls1"
+; LINUX64LE-NEXT:	.size	a, 5
+; LINUX64LE: b:
+; LINUX64LE-NEXT:	.asciz	"tls2"
+; LINUX64LE-NEXT:	.size	b, 5
+; LINUX64LE: .L__ModuleStringPool:
+; LINUX64LE-NEXT:	.long	0x42af999a
+; LINUX64LE-NEXT:	.long	0x42b1999a
+; LINUX64LE-NEXT:	.long	0x42b3cccd
+; LINUX64LE-NEXT:	.long	0x42b5999a
+; LINUX64LE-NEXT:	.long	1
+; LINUX64LE-NEXT:	.long	2
+; LINUX64LE-NEXT:	.long	3
+; LINUX64LE-NEXT:	.long	4
+; LINUX64LE-NEXT:	.long	5
+; LINUX64LE-NEXT:	.long	6
+; LINUX64LE-NEXT:	.long	7
+; LINUX64LE-NEXT:	.long	8
+; LINUX64LE-NEXT:	.long	9
+; LINUX64LE-NEXT:	.long	10
+; LINUX64LE-NEXT:	.asciz	"Regular global"
+; LINUX64LE-NEXT:	.asciz	"TLS variable 1, 2 and non-TLS var: %s, %s, %s\n"
+
+; LINUX64BE: a:
+; LINUX64BE-NEXT:	.asciz	"tls1"
+; LINUX64BE-NEXT:	.size	a, 5
+; LINUX64BE: b:
+; LINUX64BE-NEXT:	.asciz	"tls2"
+; LINUX64BE-NEXT:	.size	b, 5
+; LINUX64BE: .L__ModuleStringPool:
+; LINUX64BE-NEXT:	.long	0x42af999a
+; LINUX64BE-NEXT:	.long	0x42b1999a
+; LINUX64BE-NEXT:	.long	0x42b3cccd
+; LINUX64BE-NEXT:	.long	0x42b5999a
+; LINUX64BE-NEXT:	.long	1
+; LINUX64BE-NEXT:	.long	2
+; LINUX64BE-NEXT:	.long	3
+; LINUX64BE-NEXT:	.long	4
+; LINUX64BE-NEXT:	.long	5
+; LINUX64BE-NEXT:	.long	6
+; LINUX64BE-NEXT:	.long	7
+; LINUX64BE-NEXT:	.long	8
+; LINUX64BE-NEXT:	.long	9
+; LINUX64BE-NEXT:	.long	10
+; LINUX64BE-NEXT:	.asciz	"Regular global"
+; LINUX64BE-NEXT:	.asciz	"TLS variable 1, 2 and non-TLS var: %s, %s, %s\n"
diff --git a/llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll b/llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll
index 81147d10cde6e7d..833ed9fa65acf18 100644
--- a/llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll
+++ b/llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr8 \
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr8 -enable-global-merge=false \
 ; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s --check-prefixes=AIX32,AIXDATA
-; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr8 \
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr8 -enable-global-merge=false \
 ; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s --check-prefixes=AIX64,AIXDATA
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-unknown-linux -mcpu=pwr8 \
 ; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s --check-prefixes=LINUX64BE,LINUXDATA
diff --git a/llvm/test/CodeGen/PowerPC/noredzone.ll b/llvm/test/CodeGen/PowerPC/noredzone.ll
new file mode 100644
index 000000000000000..797a15569d615dd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/noredzone.ll
@@ -0,0 +1,519 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-ibm-aix < %s | FileCheck -check-prefix=AIX64 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-ibm-aix < %s | FileCheck -check-prefix=AIX32 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck -check-prefix=LE64 %s
+
+define signext i32 @leaf1_noredzone(i32 signext %a, i32 signext %b) #0 {
+; AIX64-LABEL: leaf1_noredzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -64(1)
+; AIX64-NEXT:    stw 3, 60(1)
+; AIX64-NEXT:    add 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 4, 56(1)
+; AIX64-NEXT:    addi 1, 1, 64
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: leaf1_noredzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    stw 3, 28(1)
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 4, 24(1)
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: leaf1_noredzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    stw 3, 44(1)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 40(1)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    blr
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define void @nonleaf1_noredzone(i32 signext %a, i32 signext %b) #0 {
+; AIX64-LABEL: nonleaf1_noredzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    stdu 1, -128(1)
+; AIX64-NEXT:    std 0, 144(1)
+; AIX64-NEXT:    stw 3, 124(1)
+; AIX64-NEXT:    add 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 4, 120(1)
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    lwz 3, 124(1)
+; AIX64-NEXT:    lwz 4, 120(1)
+; AIX64-NEXT:    sub 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    addi 1, 1, 128
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: nonleaf1_noredzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stwu 1, -64(1)
+; AIX32-NEXT:    stw 0, 72(1)
+; AIX32-NEXT:    stw 3, 60(1)
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 4, 56(1)
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 60(1)
+; AIX32-NEXT:    lwz 4, 56(1)
+; AIX32-NEXT:    sub 3, 3, 4
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    addi 1, 1, 64
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: nonleaf1_noredzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    mflr 0
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    std 0, 64(1)
+; LE64-NEXT:    stw 3, 44(1)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    stw 4, 40(1)
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    lwz 3, 44(1)
+; LE64-NEXT:    lwz 4, 40(1)
+; LE64-NEXT:    sub 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    ld 0, 16(1)
+; LE64-NEXT:    mtlr 0
+; LE64-NEXT:    blr
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  call void @leaf2(i32 signext %add)
+  %2 = load i32, ptr %a.addr, align 4
+  %3 = load i32, ptr %b.addr, align 4
+  %sub = sub nsw i32 %2, %3
+  call void @leaf2(i32 signext %sub)
+  ret void
+}
+
+declare void @leaf2(i32 signext)
+
+define signext i32 @leaf3_noredzone(i32 signext %a, i32 signext %b) #0 {
+; AIX64-LABEL: leaf3_noredzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -48(1)
+; AIX64-NEXT:    ld 6, 0(1)
+; AIX64-NEXT:    mr 5, 3
+; AIX64-NEXT:    add 3, 5, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 5, 48(6)
+; AIX64-NEXT:    stw 4, 52(6)
+; AIX64-NEXT:    addi 1, 1, 48
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: leaf3_noredzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    lwz 6, 0(1)
+; AIX32-NEXT:    mr 5, 3
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 5, 24(6)
+; AIX32-NEXT:    stw 4, 28(6)
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: leaf3_noredzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    stdu 1, -32(1)
+; LE64-NEXT:    ld 5, 0(1)
+; LE64-NEXT:    stw 3, 48(5)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 52(5)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    addi 1, 1, 32
+; LE64-NEXT:    blr
+entry:
+  %f.addr = call ptr @llvm.frameaddress(i32 1)
+  %a.addr = getelementptr ptr, ptr %f.addr, i32 6
+  %b.addr = getelementptr i32, ptr %a.addr, i32 1
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define void @nonleaf2_noredzone(i32 signext %a, i32 signext %b) #0 {
+; AIX64-LABEL: nonleaf2_noredzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    stdu 1, -128(1)
+; AIX64-NEXT:    std 0, 144(1)
+; AIX64-NEXT:    std 31, 120(1) # 8-byte Folded Spill
+; AIX64-NEXT:    add 5, 3, 4
+; AIX64-NEXT:    ld 31, 0(1)
+; AIX64-NEXT:    extsw 5, 5
+; AIX64-NEXT:    stw 3, 48(31)
+; AIX64-NEXT:    mr 3, 5
+; AIX64-NEXT:    stw 4, 52(31)
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    lwz 3, 48(31)
+; AIX64-NEXT:    lwz 4, 52(31)
+; AIX64-NEXT:    sub 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    ld 31, 120(1) # 8-byte Folded Reload
+; AIX64-NEXT:    addi 1, 1, 128
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: nonleaf2_noredzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stwu 1, -64(1)
+; AIX32-NEXT:    stw 0, 72(1)
+; AIX32-NEXT:    add 5, 3, 4
+; AIX32-NEXT:    stw 31, 60(1) # 4-byte Folded Spill
+; AIX32-NEXT:    lwz 31, 0(1)
+; AIX32-NEXT:    stw 3, 24(31)
+; AIX32-NEXT:    mr 3, 5
+; AIX32-NEXT:    stw 4, 28(31)
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 24(31)
+; AIX32-NEXT:    lwz 4, 28(31)
+; AIX32-NEXT:    sub 3, 3, 4
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 31, 60(1) # 4-byte Folded Reload
+; AIX32-NEXT:    addi 1, 1, 64
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: nonleaf2_noredzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    mflr 0
+; LE64-NEXT:    std 30, -16(1) # 8-byte Folded Spill
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    std 0, 64(1)
+; LE64-NEXT:    ld 30, 0(1)
+; LE64-NEXT:    stw 3, 48(30)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 52(30)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    lwz 3, 48(30)
+; LE64-NEXT:    lwz 4, 52(30)
+; LE64-NEXT:    sub 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    ld 0, 16(1)
+; LE64-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
+; LE64-NEXT:    mtlr 0
+; LE64-NEXT:    blr
+entry:
+  %f.addr = call ptr @llvm.frameaddress(i32 1)
+  %a.addr = getelementptr ptr, ptr %f.addr, i32 6
+  %b.addr = getelementptr i32, ptr %a.addr, i32 1
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  call void @leaf2(i32 signext %add)
+  %2 = load i32, ptr %a.addr, align 4
+  %3 = load i32, ptr %b.addr, align 4
+  %sub = sub nsw i32 %2, %3
+  call void @leaf2(i32 signext %sub)
+  ret void
+}
+
+define signext i32 @leaf1_redzone(i32 signext %a, i32 signext %b) #1 {
+; AIX64-LABEL: leaf1_redzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stw 3, -4(1)
+; AIX64-NEXT:    add 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 4, -8(1)
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: leaf1_redzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stw 3, -4(1)
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 4, -8(1)
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: leaf1_redzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    stw 3, -4(1)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, -8(1)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    blr
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define void @nonleaf1_redzone(i32 signext %a, i32 signext %b) #1 {
+; AIX64-LABEL: nonleaf1_redzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    stdu 1, -128(1)
+; AIX64-NEXT:    std 0, 144(1)
+; AIX64-NEXT:    stw 3, 124(1)
+; AIX64-NEXT:    add 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 4, 120(1)
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    lwz 3, 124(1)
+; AIX64-NEXT:    lwz 4, 120(1)
+; AIX64-NEXT:    sub 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    addi 1, 1, 128
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: nonleaf1_redzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stwu 1, -64(1)
+; AIX32-NEXT:    stw 0, 72(1)
+; AIX32-NEXT:    stw 3, 60(1)
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 4, 56(1)
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 60(1)
+; AIX32-NEXT:    lwz 4, 56(1)
+; AIX32-NEXT:    sub 3, 3, 4
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    addi 1, 1, 64
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: nonleaf1_redzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    mflr 0
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    std 0, 64(1)
+; LE64-NEXT:    stw 3, 44(1)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    stw 4, 40(1)
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    lwz 3, 44(1)
+; LE64-NEXT:    lwz 4, 40(1)
+; LE64-NEXT:    sub 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    ld 0, 16(1)
+; LE64-NEXT:    mtlr 0
+; LE64-NEXT:    blr
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  call void @leaf2(i32 signext %add)
+  %2 = load i32, ptr %a.addr, align 4
+  %3 = load i32, ptr %b.addr, align 4
+  %sub = sub nsw i32 %2, %3
+  call void @leaf2(i32 signext %sub)
+  ret void
+}
+
+define signext i32 @leaf3_redzone(i32 signext %a, i32 signext %b) #1 {
+; AIX64-LABEL: leaf3_redzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -48(1)
+; AIX64-NEXT:    ld 6, 0(1)
+; AIX64-NEXT:    mr 5, 3
+; AIX64-NEXT:    add 3, 5, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 5, 48(6)
+; AIX64-NEXT:    stw 4, 52(6)
+; AIX64-NEXT:    addi 1, 1, 48
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: leaf3_redzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    lwz 6, 0(1)
+; AIX32-NEXT:    mr 5, 3
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 5, 24(6)
+; AIX32-NEXT:    stw 4, 28(6)
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: leaf3_redzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    stdu 1, -32(1)
+; LE64-NEXT:    ld 5, 0(1)
+; LE64-NEXT:    stw 3, 48(5)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 52(5)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    addi 1, 1, 32
+; LE64-NEXT:    blr
+entry:
+  %f.addr = call ptr @llvm.frameaddress(i32 1)
+  %a.addr = getelementptr ptr, ptr %f.addr, i32 6
+  %b.addr = getelementptr i32, ptr %a.addr, i32 1
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define void @nonleaf2_redzone(i32 signext %a, i32 signext %b) #1 {
+; AIX64-LABEL: nonleaf2_redzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    stdu 1, -128(1)
+; AIX64-NEXT:    std 0, 144(1)
+; AIX64-NEXT:    std 31, 120(1) # 8-byte Folded Spill
+; AIX64-NEXT:    add 5, 3, 4
+; AIX64-NEXT:    ld 31, 0(1)
+; AIX64-NEXT:    extsw 5, 5
+; AIX64-NEXT:    stw 3, 48(31)
+; AIX64-NEXT:    mr 3, 5
+; AIX64-NEXT:    stw 4, 52(31)
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    lwz 3, 48(31)
+; AIX64-NEXT:    lwz 4, 52(31)
+; AIX64-NEXT:    sub 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    ld 31, 120(1) # 8-byte Folded Reload
+; AIX64-NEXT:    addi 1, 1, 128
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: nonleaf2_redzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stwu 1, -64(1)
+; AIX32-NEXT:    stw 0, 72(1)
+; AIX32-NEXT:    add 5, 3, 4
+; AIX32-NEXT:    stw 31, 60(1) # 4-byte Folded Spill
+; AIX32-NEXT:    lwz 31, 0(1)
+; AIX32-NEXT:    stw 3, 24(31)
+; AIX32-NEXT:    mr 3, 5
+; AIX32-NEXT:    stw 4, 28(31)
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 24(31)
+; AIX32-NEXT:    lwz 4, 28(31)
+; AIX32-NEXT:    sub 3, 3, 4
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 31, 60(1) # 4-byte Folded Reload
+; AIX32-NEXT:    addi 1, 1, 64
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: nonleaf2_redzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    mflr 0
+; LE64-NEXT:    std 30, -16(1) # 8-byte Folded Spill
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    std 0, 64(1)
+; LE64-NEXT:    ld 30, 0(1)
+; LE64-NEXT:    stw 3, 48(30)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 52(30)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    lwz 3, 48(30)
+; LE64-NEXT:    lwz 4, 52(30)
+; LE64-NEXT:    sub 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    ld 0, 16(1)
+; LE64-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
+; LE64-NEXT:    mtlr 0
+; LE64-NEXT:    blr
+entry:
+  %f.addr = call ptr @llvm.frameaddress(i32 1)
+  %a.addr = getelementptr ptr, ptr %f.addr, i32 6
+  %b.addr = getelementptr i32, ptr %a.addr, i32 1
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  call void @leaf2(i32 signext %add)
+  %2 = load i32, ptr %a.addr, align 4
+  %3 = load i32, ptr %b.addr, align 4
+  %sub = sub nsw i32 %2, %3
+  call void @leaf2(i32 signext %sub)
+  ret void
+}
+
+attributes #0 = { noredzone nounwind noinline }
+attributes #1 = { nounwind noinline }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv32.mir
new file mode 100644
index 000000000000000..d8fe9b77de432d9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv32.mir
@@ -0,0 +1,79 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            vload_nxv1ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv1ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 1 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 1 x p0>))
+    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 1 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 1 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 1 x p0>), align 4)
+    $v8 = COPY %1(<vscale x 1 x p0>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            vload_nxv2ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv2ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 2 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 2 x p0>))
+    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 2 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 2 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 2 x p0>), align 8)
+    $v8 = COPY %1(<vscale x 2 x p0>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            vload_nxv8ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv8ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 8 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 8 x p0>))
+    ; CHECK-NEXT: $v8m4 = COPY [[LOAD]](<vscale x 8 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 8 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 8 x p0>), align 32)
+    $v8m4 = COPY %1(<vscale x 8 x p0>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            vload_nxv16ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv16ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 16 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 16 x p0>))
+    ; CHECK-NEXT: $v8m4 = COPY [[LOAD]](<vscale x 16 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 16 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 16 x p0>), align 64)
+    $v8m4 = COPY %1(<vscale x 16 x p0>)
+    PseudoRET implicit $v8m8
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv64.mir
new file mode 100644
index 000000000000000..98dee70a42cdba6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv64.mir
@@ -0,0 +1,60 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            vload_nxv1ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv1ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 1 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 1 x p0>))
+    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 1 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 1 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 1 x p0>), align 8)
+    $v8 = COPY %1(<vscale x 1 x p0>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            vload_nxv2ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv2ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 2 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 2 x p0>))
+    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 2 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 2 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 2 x p0>), align 16)
+    $v8 = COPY %1(<vscale x 2 x p0>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            vload_nxv8ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv8ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 8 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 8 x p0>))
+    ; CHECK-NEXT: $v8m4 = COPY [[LOAD]](<vscale x 8 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 8 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 8 x p0>), align 64)
+    $v8m4 = COPY %1(<vscale x 8 x p0>)
+    PseudoRET implicit $v8m8
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load.mir
index 12f218863e4002b..2023bf7c05565b8 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load.mir
@@ -203,21 +203,6 @@
     ret <vscale x 2 x i64> %va
   }
 
-  define <vscale x 1 x ptr> @vload_nxv1ptr(ptr %pa) #0 {
-    %va = load <vscale x 1 x ptr>, ptr %pa, align 4
-    ret <vscale x 1 x ptr> %va
-  }
-
-  define <vscale x 2 x ptr> @vload_nxv2ptr(ptr %pa) #0 {
-    %va = load <vscale x 2 x ptr>, ptr %pa, align 8
-    ret <vscale x 2 x ptr> %va
-  }
-
-  define <vscale x 8 x ptr> @vload_nxv8ptr(ptr %pa) #0 {
-    %va = load <vscale x 8 x ptr>, ptr %pa, align 32
-    ret <vscale x 8 x ptr> %va
-  }
-
   attributes #0 = { "target-features"="+v" }
 
 ...
@@ -984,60 +969,3 @@ body:             |
     PseudoRET implicit $v8m2
 
 ...
----
-name:            vload_nxv1ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10
-
-    ; CHECK-LABEL: name: vload_nxv1ptr
-    ; CHECK: liveins: $x10
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 1 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 1 x p0>) from %ir.pa)
-    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 1 x p0>)
-    ; CHECK-NEXT: PseudoRET implicit $v8
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 1 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 1 x p0>) from %ir.pa)
-    $v8 = COPY %1(<vscale x 1 x p0>)
-    PseudoRET implicit $v8
-
-...
----
-name:            vload_nxv2ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10
-
-    ; CHECK-LABEL: name: vload_nxv2ptr
-    ; CHECK: liveins: $x10
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 2 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 2 x p0>) from %ir.pa)
-    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 2 x p0>)
-    ; CHECK-NEXT: PseudoRET implicit $v8
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 2 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 2 x p0>) from %ir.pa)
-    $v8 = COPY %1(<vscale x 2 x p0>)
-    PseudoRET implicit $v8
-
-...
----
-name:            vload_nxv8ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10
-
-    ; CHECK-LABEL: name: vload_nxv8ptr
-    ; CHECK: liveins: $x10
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 8 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 8 x p0>) from %ir.pa)
-    ; CHECK-NEXT: $v8m4 = COPY [[LOAD]](<vscale x 8 x p0>)
-    ; CHECK-NEXT: PseudoRET implicit $v8m4
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 8 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 8 x p0>) from %ir.pa)
-    $v8m4 = COPY %1(<vscale x 8 x p0>)
-    PseudoRET implicit $v8m4
-
-...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv32.mir
new file mode 100644
index 000000000000000..a93cce6904c2ca8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv32.mir
@@ -0,0 +1,79 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            vstore_nx1ptr
+body:             |
+  bb.1:
+    liveins: $v8, $x10
+
+    ; CHECK-LABEL: name: vstore_nx1ptr
+    ; CHECK: liveins: $v8, $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 1 x p0>) = COPY $v8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 1 x p0>), [[COPY]](p0) :: (store (<vscale x 1 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 1 x p0>) = COPY $v8
+    G_STORE %1(<vscale x 1 x p0>), %0(p0) :: (store (<vscale x 1 x p0>), align 4)
+    PseudoRET
+
+...
+---
+name:            vstore_nx2ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8
+
+    ; CHECK-LABEL: name: vstore_nx2ptr
+    ; CHECK: liveins: $x10, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 2 x p0>) = COPY $v8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 2 x p0>), [[COPY]](p0) :: (store (<vscale x 2 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 2 x p0>) = COPY $v8
+    G_STORE %1(<vscale x 2 x p0>), %0(p0) :: (store (<vscale x 2 x p0>), align 8)
+    PseudoRET
+
+...
+---
+name:            vstore_nx8ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8m4
+
+    ; CHECK-LABEL: name: vstore_nx8ptr
+    ; CHECK: liveins: $x10, $v8m4
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 8 x p0>) = COPY $v8m4
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 8 x p0>), [[COPY]](p0) :: (store (<vscale x 8 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 8 x p0>) = COPY $v8m4
+    G_STORE %1(<vscale x 8 x p0>), %0(p0) :: (store (<vscale x 8 x p0>), align 32)
+    PseudoRET
+
+...
+---
+name:            vstore_nx16ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8m8
+
+    ; CHECK-LABEL: name: vstore_nx16ptr
+    ; CHECK: liveins: $x10, $v8m8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 16 x p0>) = COPY $v8m8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 16 x p0>), [[COPY]](p0) :: (store (<vscale x 8 x p0>), align 64)
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 16 x p0>) = COPY $v8m8
+    G_STORE %1(<vscale x 16 x p0>), %0(p0) :: (store (<vscale x 8 x p0>), align 64)
+    PseudoRET
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv64.mir
new file mode 100644
index 000000000000000..3373ed4d287467f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv64.mir
@@ -0,0 +1,60 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            vstore_nx1ptr
+body:             |
+  bb.1:
+    liveins: $v8, $x10
+
+    ; CHECK-LABEL: name: vstore_nx1ptr
+    ; CHECK: liveins: $v8, $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 1 x p0>) = COPY $v8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 1 x p0>), [[COPY]](p0) :: (store (<vscale x 1 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 1 x p0>) = COPY $v8
+    G_STORE %1(<vscale x 1 x p0>), %0(p0) :: (store (<vscale x 1 x p0>), align 8)
+    PseudoRET
+
+...
+---
+name:            vstore_nx2ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8m2
+
+    ; CHECK-LABEL: name: vstore_nx2ptr
+    ; CHECK: liveins: $x10, $v8m2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 2 x p0>) = COPY $v8m2
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 2 x p0>), [[COPY]](p0) :: (store (<vscale x 2 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 2 x p0>) = COPY $v8m2
+    G_STORE %1(<vscale x 2 x p0>), %0(p0) :: (store (<vscale x 2 x p0>), align 16)
+    PseudoRET
+
+...
+---
+name:            vstore_nx8ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8m8
+
+    ; CHECK-LABEL: name: vstore_nx8ptr
+    ; CHECK: liveins: $x10, $v8m8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 8 x p0>) = COPY $v8m8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 8 x p0>), [[COPY]](p0) :: (store (<vscale x 8 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 8 x p0>) = COPY $v8m8
+    G_STORE %1(<vscale x 8 x p0>), %0(p0) :: (store (<vscale x 8 x p0>), align 64)
+    PseudoRET
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store.mir
index b91d25509646f5d..4bb4eb5fa0c72b3 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store.mir
@@ -203,21 +203,6 @@
     ret void
   }
 
-  define void @vstore_nx1ptr(ptr %pa, <vscale x 1 x ptr> %b) #0 {
-    store <vscale x 1 x ptr> %b, ptr %pa, align 4
-    ret void
-  }
-
-  define void @vstore_nx2ptr(ptr %pa, <vscale x 2 x ptr> %b) #0 {
-    store <vscale x 2 x ptr> %b, ptr %pa, align 8
-    ret void
-  }
-
-  define void @vstore_nx8ptr(ptr %pa, <vscale x 8 x ptr> %b) #0 {
-    store <vscale x 8 x ptr> %b, ptr %pa, align 32
-    ret void
-  }
-
   attributes #0 = { "target-features"="+v" }
 
 ...
@@ -984,60 +969,3 @@ body:             |
     PseudoRET
 
 ...
----
-name:            vstore_nx1ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $v8, $x10
-
-    ; CHECK-LABEL: name: vstore_nx1ptr
-    ; CHECK: liveins: $v8, $x10
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 1 x p0>) = COPY $v8
-    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 1 x p0>), [[COPY]](p0) :: (store (<vscale x 1 x p0>) into %ir.pa)
-    ; CHECK-NEXT: PseudoRET
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 1 x p0>) = COPY $v8
-    G_STORE %1(<vscale x 1 x p0>), %0(p0) :: (store (<vscale x 1 x p0>) into %ir.pa)
-    PseudoRET
-
-...
----
-name:            vstore_nx2ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10, $v8m2
-
-    ; CHECK-LABEL: name: vstore_nx2ptr
-    ; CHECK: liveins: $x10, $v8m2
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 2 x p0>) = COPY $v8m2
-    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 2 x p0>), [[COPY]](p0) :: (store (<vscale x 2 x p0>) into %ir.pa)
-    ; CHECK-NEXT: PseudoRET
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 2 x p0>) = COPY $v8m2
-    G_STORE %1(<vscale x 2 x p0>), %0(p0) :: (store (<vscale x 2 x p0>) into %ir.pa)
-    PseudoRET
-
-...
----
-name:            vstore_nx8ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10, $v8m8
-
-    ; CHECK-LABEL: name: vstore_nx8ptr
-    ; CHECK: liveins: $x10, $v8m8
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 8 x p0>) = COPY $v8m8
-    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 8 x p0>), [[COPY]](p0) :: (store (<vscale x 8 x p0>) into %ir.pa)
-    ; CHECK-NEXT: PseudoRET
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 8 x p0>) = COPY $v8m8
-    G_STORE %1(<vscale x 8 x p0>), %0(p0) :: (store (<vscale x 8 x p0>) into %ir.pa)
-    PseudoRET
-
-...
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index df9cb5de5d76825..44c270fdc3c2577 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -45,6 +45,7 @@
 ; CHECK-NEXT:         Canonicalize Freeze Instructions in Loops
 ; CHECK-NEXT:         Induction Variable Users
 ; CHECK-NEXT:         Loop Strength Reduction
+; CHECK-NEXT:         Loop Terminator Folding
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Merge contiguous icmps into a memcmp
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 4f841683a868c4e..3aee484beeaa357 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -125,6 +125,8 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+zama16b %s -o - | FileCheck --check-prefixes=CHECK,RV32ZAMA16B %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zicfilp %s -o - | FileCheck --check-prefix=RV32ZICFILP %s
 ; RUN: llc -mtriple=riscv32 -mattr=+a,+zabha %s -o - | FileCheck --check-prefix=RV32ZABHA %s
+; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvbc32e  %s -o - | FileCheck --check-prefix=RV32ZVBC32E %s
+; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvkgs  %s -o - | FileCheck --check-prefix=RV32ZVKGS %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-ssnpm  %s -o - | FileCheck --check-prefix=RV32SSNPM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-smnpm  %s -o - | FileCheck --check-prefix=RV32SMNPM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-smmpm %s -o - | FileCheck --check-prefix=RV32SMMPM %s
@@ -263,6 +265,8 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zalasr %s -o - | FileCheck --check-prefix=RV64ZALASR %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zicfilp %s -o - | FileCheck --check-prefix=RV64ZICFILP %s
 ; RUN: llc -mtriple=riscv64 -mattr=+a,+zabha %s -o - | FileCheck --check-prefix=RV64ZABHA %s
+; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvbc32e  %s -o - | FileCheck --check-prefix=RV64ZVBC32E %s
+; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvkgs  %s -o - | FileCheck --check-prefix=RV64ZVKGS %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-ssnpm  %s -o - | FileCheck --check-prefix=RV64SSNPM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-smnpm  %s -o - | FileCheck --check-prefix=RV64SMNPM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-smmpm %s -o - | FileCheck --check-prefix=RV64SMMPM %s
@@ -409,6 +413,8 @@
 ; RV32ZAMA16B: .attribute 5, "rv32i2p1_zama16b1p0"
 ; RV32ZICFILP: .attribute 5, "rv32i2p1_zicfilp1p0_zicsr2p0"
 ; RV32ZABHA: .attribute 5, "rv32i2p1_a2p1_zabha1p0"
+; RV32ZVBC32E: .attribute 5, "rv32i2p1_zicsr2p0_zvbc32e0p7_zve32x1p0_zvl32b1p0"
+; RV32ZVKGS: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvkg1p0_zvkgs0p7_zvl32b1p0"
 ; RV32SSNPM: .attribute 5, "rv32i2p1_ssnpm1p0"
 ; RV32SMNPM: .attribute 5, "rv32i2p1_smnpm1p0"
 ; RV32SMMPM: .attribute 5, "rv32i2p1_smmpm1p0"
@@ -545,6 +551,8 @@
 ; RV64ZALASR: .attribute 5, "rv64i2p1_zalasr0p1"
 ; RV64ZICFILP: .attribute 5, "rv64i2p1_zicfilp1p0_zicsr2p0"
 ; RV64ZABHA: .attribute 5, "rv64i2p1_a2p1_zabha1p0"
+; RV64ZVBC32E: .attribute 5, "rv64i2p1_zicsr2p0_zvbc32e0p7_zve32x1p0_zvl32b1p0"
+; RV64ZVKGS: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvkg1p0_zvkgs0p7_zvl32b1p0"
 ; RV64SSNPM: .attribute 5, "rv64i2p1_ssnpm1p0"
 ; RV64SMNPM: .attribute 5, "rv64i2p1_smnpm1p0"
 ; RV64SMMPM: .attribute 5, "rv64i2p1_smmpm1p0"
diff --git a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
index 8caa64c9572ce78..2c691a2de4c4de7 100644
--- a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
@@ -2367,10 +2367,9 @@ define i16 @test_ctpop_i16(i16 %a) nounwind {
 ; RV32XTHEADBB-NEXT:    add a0, a2, a0
 ; RV32XTHEADBB-NEXT:    srli a1, a0, 4
 ; RV32XTHEADBB-NEXT:    add a0, a0, a1
-; RV32XTHEADBB-NEXT:    andi a1, a0, 15
-; RV32XTHEADBB-NEXT:    slli a0, a0, 20
-; RV32XTHEADBB-NEXT:    srli a0, a0, 28
-; RV32XTHEADBB-NEXT:    add a0, a1, a0
+; RV32XTHEADBB-NEXT:    th.extu a1, a0, 11, 8
+; RV32XTHEADBB-NEXT:    andi a0, a0, 15
+; RV32XTHEADBB-NEXT:    add a0, a0, a1
 ; RV32XTHEADBB-NEXT:    ret
 ;
 ; RV64XTHEADBB-LABEL: test_ctpop_i16:
@@ -2388,10 +2387,9 @@ define i16 @test_ctpop_i16(i16 %a) nounwind {
 ; RV64XTHEADBB-NEXT:    add a0, a2, a0
 ; RV64XTHEADBB-NEXT:    srli a1, a0, 4
 ; RV64XTHEADBB-NEXT:    add a0, a0, a1
-; RV64XTHEADBB-NEXT:    andi a1, a0, 15
-; RV64XTHEADBB-NEXT:    slli a0, a0, 52
-; RV64XTHEADBB-NEXT:    srli a0, a0, 60
-; RV64XTHEADBB-NEXT:    add a0, a1, a0
+; RV64XTHEADBB-NEXT:    th.extu a1, a0, 11, 8
+; RV64XTHEADBB-NEXT:    andi a0, a0, 15
+; RV64XTHEADBB-NEXT:    add a0, a0, a1
 ; RV64XTHEADBB-NEXT:    ret
   %1 = call i16 @llvm.ctpop.i16(i16 %a)
   ret i16 %1
diff --git a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
index 6cdab888ffcde7f..8ce4c44d8efcc2a 100644
--- a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
@@ -702,6 +702,22 @@ define i64 @zext_bf_i64(i64 %a) nounwind {
   ret i64 %and
 }
 
+define i64 @zext_bf2_i64(i64 %a) nounwind {
+; RV64I-LABEL: zext_bf2_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 49
+; RV64I-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: zext_bf2_i64:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    th.extu a0, a0, 15, 1
+; RV64XTHEADBB-NEXT:    ret
+  %t0 = and i64 %a, 65535
+  %result = lshr i64 %t0, 1
+  ret i64 %result
+}
+
 define i64 @zext_i64_srliw(i64 %a) nounwind {
 ; RV64I-LABEL: zext_i64_srliw:
 ; RV64I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse-bitrotate.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse-bitrotate.ll
deleted file mode 100644
index a81f740f1739340..000000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse-bitrotate.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zvl1024b -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zvl1024b -verify-machineinstrs < %s | FileCheck %s
-
-; There is no corresponding v1i256 type, so make sure we don't crash if we try
-; to lower via lowerBitreverseShuffle.
-define <256 x i1> @reverse_v256i1(<256 x i1> %a) {
-; CHECK-LABEL: reverse_v256i1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 256
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v10, v10, -1
-; CHECK-NEXT:    vrgather.vv v12, v8, v10
-; CHECK-NEXT:    vmsne.vi v0, v12, 0
-; CHECK-NEXT:    ret
-  %res = call <256 x i1> @llvm.vector.reverse.v256i1(<256 x i1> %a)
-  ret <256 x i1> %res
-}
-
-declare <256 x i1> @llvm.vector.reverse.v256i1(<256 x i1>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index 4b1f0beb4870084..e773d93fad47961 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -1,17 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32-BITS-256
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32-BITS-512
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64-BITS-256
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64-BITS-512
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV32-ZVBB
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV64-ZVBB
 
-;
-; VECTOR_REVERSE - masks
-;
-
 define <2 x i1> @reverse_v2i1(<2 x i1> %a) {
 ; NO-ZVBB-LABEL: reverse_v2i1:
 ; NO-ZVBB:       # %bb.0:
@@ -29,7 +21,7 @@ define <2 x i1> @reverse_v2i1(<2 x i1> %a) {
 ; ZVBB-NEXT:    vbrev.v v8, v0
 ; ZVBB-NEXT:    vsrl.vi v0, v8, 6
 ; ZVBB-NEXT:    ret
-  %res = call <2 x i1> @llvm.vector.reverse.v2i1(<2 x i1> %a)
+  %res = shufflevector <2 x i1> %a, <2 x i1> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i1> %res
 }
 
@@ -51,7 +43,7 @@ define <4 x i1> @reverse_v4i1(<4 x i1> %a) {
 ; ZVBB-NEXT:    vbrev.v v8, v0
 ; ZVBB-NEXT:    vsrl.vi v0, v8, 4
 ; ZVBB-NEXT:    ret
-  %res = call <4 x i1> @llvm.vector.reverse.v4i1(<4 x i1> %a)
+  %res = shufflevector <4 x i1> %a, <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i1> %res
 }
 
@@ -72,7 +64,7 @@ define <8 x i1> @reverse_v8i1(<8 x i1> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; ZVBB-NEXT:    vbrev.v v0, v0
 ; ZVBB-NEXT:    ret
-  %res = call <8 x i1> @llvm.vector.reverse.v8i1(<8 x i1> %a)
+  %res = shufflevector <8 x i1> %a, <8 x i1> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i1> %res
 }
 
@@ -93,7 +85,7 @@ define <16 x i1> @reverse_v16i1(<16 x i1> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; ZVBB-NEXT:    vbrev.v v0, v0
 ; ZVBB-NEXT:    ret
-  %res = call <16 x i1> @llvm.vector.reverse.v16i1(<16 x i1> %a)
+  %res = shufflevector <16 x i1> %a, <16 x i1> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i1> %res
 }
 
@@ -101,10 +93,10 @@ define <32 x i1> @reverse_v32i1(<32 x i1> %a) {
 ; NO-ZVBB-LABEL: reverse_v32i1:
 ; NO-ZVBB:       # %bb.0:
 ; NO-ZVBB-NEXT:    li a0, 32
-; NO-ZVBB-NEXT:    lui a1, %hi(.LCPI4_0)
-; NO-ZVBB-NEXT:    addi a1, a1, %lo(.LCPI4_0)
 ; NO-ZVBB-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; NO-ZVBB-NEXT:    vle8.v v8, (a1)
+; NO-ZVBB-NEXT:    vid.v v8
+; NO-ZVBB-NEXT:    li a0, 31
+; NO-ZVBB-NEXT:    vrsub.vx v8, v8, a0
 ; NO-ZVBB-NEXT:    vmv.v.i v10, 0
 ; NO-ZVBB-NEXT:    vmerge.vim v10, v10, 1, v0
 ; NO-ZVBB-NEXT:    vrgather.vv v12, v10, v8
@@ -116,7 +108,7 @@ define <32 x i1> @reverse_v32i1(<32 x i1> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vbrev.v v0, v0
 ; ZVBB-NEXT:    ret
-  %res = call <32 x i1> @llvm.vector.reverse.v32i1(<32 x i1> %a)
+  %res = shufflevector <32 x i1> %a, <32 x i1> poison, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i1> %res
 }
 
@@ -124,10 +116,10 @@ define <64 x i1> @reverse_v64i1(<64 x i1> %a) {
 ; NO-ZVBB-LABEL: reverse_v64i1:
 ; NO-ZVBB:       # %bb.0:
 ; NO-ZVBB-NEXT:    li a0, 64
-; NO-ZVBB-NEXT:    lui a1, %hi(.LCPI5_0)
-; NO-ZVBB-NEXT:    addi a1, a1, %lo(.LCPI5_0)
 ; NO-ZVBB-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; NO-ZVBB-NEXT:    vle8.v v8, (a1)
+; NO-ZVBB-NEXT:    vid.v v8
+; NO-ZVBB-NEXT:    li a0, 63
+; NO-ZVBB-NEXT:    vrsub.vx v8, v8, a0
 ; NO-ZVBB-NEXT:    vmv.v.i v12, 0
 ; NO-ZVBB-NEXT:    vmerge.vim v12, v12, 1, v0
 ; NO-ZVBB-NEXT:    vrgather.vv v16, v12, v8
@@ -139,7 +131,7 @@ define <64 x i1> @reverse_v64i1(<64 x i1> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; ZVBB-NEXT:    vbrev.v v0, v0
 ; ZVBB-NEXT:    ret
-  %res = call <64 x i1> @llvm.vector.reverse.v64i1(<64 x i1> %a)
+  %res = shufflevector <64 x i1> %a, <64 x i1> poison, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <64 x i1> %res
 }
 
@@ -147,16 +139,16 @@ define <128 x i1> @reverse_v128i1(<128 x i1> %a) {
 ; CHECK-LABEL: reverse_v128i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 128
-; CHECK-NEXT:    lui a1, %hi(.LCPI6_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI6_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    li a0, 127
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
 ; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
 ; CHECK-NEXT:    vrgather.vv v24, v16, v8
 ; CHECK-NEXT:    vmsne.vi v0, v24, 0
 ; CHECK-NEXT:    ret
-  %res = call <128 x i1> @llvm.vector.reverse.v128i1(<128 x i1> %a)
+  %res = shufflevector <128 x i1> %a, <128 x i1> poison, <128 x i32> <i32 127, i32 126, i32 125, i32 124, i32 123, i32 122, i32 121, i32 120, i32 119, i32 118, i32 117, i32 116, i32 115, i32 114, i32 113, i32 112, i32 111, i32 110, i32 109, i32 108, i32 107, i32 106, i32 105, i32 104, i32 103, i32 102, i32 101, i32 100, i32 99, i32 98, i32 97, i32 96, i32 95, i32 94, i32 93, i32 92, i32 91, i32 90, i32 89, i32 88, i32 87, i32 86, i32 85, i32 84, i32 83, i32 82, i32 81, i32 80, i32 79, i32 78, i32 77, i32 76, i32 75, i32 74, i32 73, i32 72, i32 71, i32 70, i32 69, i32 68, i32 67, i32 66, i32 65, i32 64, i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <128 x i1> %res
 }
 
@@ -164,7 +156,7 @@ define <1 x i8> @reverse_v1i8(<1 x i8> %a) {
 ; CHECK-LABEL: reverse_v1i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x i8> @llvm.vector.reverse.v1i8(<1 x i8> %a)
+  %res = shufflevector <1 x i8> %a, <1 x i8> poison, <1 x i32> <i32 0>
   ret <1 x i8> %res
 }
 
@@ -182,7 +174,7 @@ define <2 x i8> @reverse_v2i8(<2 x i8> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; ZVBB-NEXT:    vrev8.v v8, v8
 ; ZVBB-NEXT:    ret
-  %res = call <2 x i8> @llvm.vector.reverse.v2i8(<2 x i8> %a)
+  %res = shufflevector <2 x i8> %a, <2 x i8> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i8> %res
 }
 
@@ -195,7 +187,7 @@ define <4 x i8> @reverse_v4i8(<4 x i8> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x i8> @llvm.vector.reverse.v4i8(<4 x i8> %a)
+  %res = shufflevector <4 x i8> %a, <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i8> %res
 }
 
@@ -208,7 +200,7 @@ define <8 x i8> @reverse_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <8 x i8> @llvm.vector.reverse.v8i8(<8 x i8> %a)
+  %res = shufflevector <8 x i8> %a, <8 x i8> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i8> %res
 }
 
@@ -221,7 +213,7 @@ define <16 x i8> @reverse_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <16 x i8> @llvm.vector.reverse.v16i8(<16 x i8> %a)
+  %res = shufflevector <16 x i8> %a, <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i8> %res
 }
 
@@ -229,14 +221,14 @@ define <32 x i8> @reverse_v32i8(<32 x i8> %a) {
 ; CHECK-LABEL: reverse_v32i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI12_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI12_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v12, v10, a0
 ; CHECK-NEXT:    vrgather.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <32 x i8> @llvm.vector.reverse.v32i8(<32 x i8> %a)
+  %res = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i8> %res
 }
 
@@ -244,14 +236,14 @@ define <64 x i8> @reverse_v64i8(<64 x i8> %a) {
 ; CHECK-LABEL: reverse_v64i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 64
-; CHECK-NEXT:    lui a1, %hi(.LCPI13_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI13_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vle8.v v16, (a1)
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 63
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <64 x i8> @llvm.vector.reverse.v64i8(<64 x i8> %a)
+  %res = shufflevector <64 x i8> %a, <64 x i8> poison, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <64 x i8> %res
 }
 
@@ -259,7 +251,7 @@ define <1 x i16> @reverse_v1i16(<1 x i16> %a) {
 ; CHECK-LABEL: reverse_v1i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x i16> @llvm.vector.reverse.v1i16(<1 x i16> %a)
+  %res = shufflevector <1 x i16> %a, <1 x i16> poison, <1 x i32> <i32 0>
   ret <1 x i16> %res
 }
 
@@ -277,7 +269,7 @@ define <2 x i16> @reverse_v2i16(<2 x i16> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vror.vi v8, v8, 16
 ; ZVBB-NEXT:    ret
-  %res = call <2 x i16> @llvm.vector.reverse.v2i16(<2 x i16> %a)
+  %res = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i16> %res
 }
 
@@ -290,7 +282,7 @@ define <4 x i16> @reverse_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x i16> @llvm.vector.reverse.v4i16(<4 x i16> %a)
+  %res = shufflevector <4 x i16> %a, <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i16> %res
 }
 
@@ -303,7 +295,7 @@ define <8 x i16> @reverse_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <8 x i16> @llvm.vector.reverse.v8i16(<8 x i16> %a)
+  %res = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i16> %res
 }
 
@@ -316,7 +308,7 @@ define <16 x i16> @reverse_v16i16(<16 x i16> %a) {
 ; CHECK-NEXT:    vrgather.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <16 x i16> @llvm.vector.reverse.v16i16(<16 x i16> %a)
+  %res = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i16> %res
 }
 
@@ -324,15 +316,14 @@ define <32 x i16> @reverse_v32i16(<32 x i16> %a) {
 ; CHECK-LABEL: reverse_v32i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI19_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI19_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
-; CHECK-NEXT:    vsext.vf2 v16, v12
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <32 x i16> @llvm.vector.reverse.v32i16(<32 x i16> %a)
+  %res = shufflevector <32 x i16> %a, <32 x i16> poison, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i16> %res
 }
 
@@ -340,7 +331,7 @@ define <1 x i32> @reverse_v1i32(<1 x i32> %a) {
 ; CHECK-LABEL: reverse_v1i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x i32> @llvm.vector.reverse.v1i32(<1 x i32> %a)
+  %res = shufflevector <1 x i32> %a, <1 x i32> poison, <1 x i32> <i32 0>
   ret <1 x i32> %res
 }
 
@@ -358,7 +349,7 @@ define <2 x i32> @reverse_v2i32(<2 x i32> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; ZVBB-NEXT:    vror.vi v8, v8, 32
 ; ZVBB-NEXT:    ret
-  %res = call <2 x i32> @llvm.vector.reverse.v2i32(<2 x i32> %a)
+  %res = shufflevector <2 x i32> %a, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i32> %res
 }
 
@@ -371,7 +362,7 @@ define <4 x i32> @reverse_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32> %a)
+  %res = shufflevector <4 x i32> %a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i32> %res
 }
 
@@ -385,7 +376,7 @@ define <8 x i32> @reverse_v8i32(<8 x i32> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <8 x i32> @llvm.vector.reverse.v8i32(<8 x i32> %a)
+  %res = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i32> %res
 }
 
@@ -399,7 +390,7 @@ define <16 x i32> @reverse_v16i32(<16 x i32> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <16 x i32> @llvm.vector.reverse.v16i32(<16 x i32> %a)
+  %res = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i32> %res
 }
 
@@ -407,7 +398,7 @@ define <1 x i64> @reverse_v1i64(<1 x i64> %a) {
 ; CHECK-LABEL: reverse_v1i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x i64> @llvm.vector.reverse.v1i64(<1 x i64> %a)
+  %res = shufflevector <1 x i64> %a, <1 x i64> poison, <1 x i32> <i32 0>
   ret <1 x i64> %res
 }
 
@@ -419,7 +410,7 @@ define <2 x i64> @reverse_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    vslideup.vi v9, v8, 1
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <2 x i64> @llvm.vector.reverse.v2i64(<2 x i64> %a)
+  %res = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i64> %res
 }
 
@@ -433,7 +424,7 @@ define <4 x i64> @reverse_v4i64(<4 x i64> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <4 x i64> @llvm.vector.reverse.v4i64(<4 x i64> %a)
+  %res = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i64> %res
 }
 
@@ -447,7 +438,7 @@ define <8 x i64> @reverse_v8i64(<8 x i64> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <8 x i64> @llvm.vector.reverse.v8i64(<8 x i64> %a)
+  %res = shufflevector <8 x i64> %a, <8 x i64> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i64> %res
 }
 
@@ -456,7 +447,7 @@ define <1 x half> @reverse_v1f16(<1 x half> %a) {
 ; CHECK-LABEL: reverse_v1f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x half> @llvm.vector.reverse.v1f16(<1 x half> %a)
+  %res = shufflevector <1 x half> %a, <1 x half> poison, <1 x i32> <i32 0>
   ret <1 x half> %res
 }
 
@@ -474,7 +465,7 @@ define <2 x half> @reverse_v2f16(<2 x half> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vror.vi v8, v8, 16
 ; ZVBB-NEXT:    ret
-  %res = call <2 x half> @llvm.vector.reverse.v2f16(<2 x half> %a)
+  %res = shufflevector <2 x half> %a, <2 x half> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x half> %res
 }
 
@@ -487,7 +478,7 @@ define <4 x half> @reverse_v4f16(<4 x half> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x half> @llvm.vector.reverse.v4f16(<4 x half> %a)
+  %res = shufflevector <4 x half> %a, <4 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x half> %res
 }
 
@@ -500,7 +491,7 @@ define <8 x half> @reverse_v8f16(<8 x half> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <8 x half> @llvm.vector.reverse.v8f16(<8 x half> %a)
+  %res = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x half> %res
 }
 
@@ -513,7 +504,7 @@ define <16 x half> @reverse_v16f16(<16 x half> %a) {
 ; CHECK-NEXT:    vrgather.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <16 x half> @llvm.vector.reverse.v16f16(<16 x half> %a)
+  %res = shufflevector <16 x half> %a, <16 x half> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x half> %res
 }
 
@@ -521,15 +512,14 @@ define <32 x half> @reverse_v32f16(<32 x half> %a) {
 ; CHECK-LABEL: reverse_v32f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI34_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI34_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
-; CHECK-NEXT:    vsext.vf2 v16, v12
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <32 x half> @llvm.vector.reverse.v32f16(<32 x half> %a)
+  %res = shufflevector <32 x half> %a, <32 x half> poison, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x half> %res
 }
 
@@ -537,7 +527,7 @@ define <1 x float> @reverse_v1f32(<1 x float> %a) {
 ; CHECK-LABEL: reverse_v1f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x float> @llvm.vector.reverse.v1f32(<1 x float> %a)
+  %res = shufflevector <1 x float> %a, <1 x float> poison, <1 x i32> <i32 0>
   ret <1 x float> %res
 }
 
@@ -555,7 +545,7 @@ define <2 x float> @reverse_v2f32(<2 x float> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; ZVBB-NEXT:    vror.vi v8, v8, 32
 ; ZVBB-NEXT:    ret
-  %res = call <2 x float> @llvm.vector.reverse.v2f32(<2 x float> %a)
+  %res = shufflevector <2 x float> %a, <2 x float> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x float> %res
 }
 
@@ -568,7 +558,7 @@ define <4 x float> @reverse_v4f32(<4 x float> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> %a)
+  %res = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x float> %res
 }
 
@@ -582,7 +572,7 @@ define <8 x float> @reverse_v8f32(<8 x float> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <8 x float> @llvm.vector.reverse.v8f32(<8 x float> %a)
+  %res = shufflevector <8 x float> %a, <8 x float> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x float> %res
 }
 
@@ -596,7 +586,7 @@ define <16 x float> @reverse_v16f32(<16 x float> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <16 x float> @llvm.vector.reverse.v16f32(<16 x float> %a)
+  %res = shufflevector <16 x float> %a, <16 x float> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x float> %res
 }
 
@@ -604,7 +594,7 @@ define <1 x double> @reverse_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: reverse_v1f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x double> @llvm.vector.reverse.v1f64(<1 x double> %a)
+  %res = shufflevector <1 x double> %a, <1 x double> poison, <1 x i32> <i32 0>
   ret <1 x double> %res
 }
 
@@ -616,7 +606,7 @@ define <2 x double> @reverse_v2f64(<2 x double> %a) {
 ; CHECK-NEXT:    vslideup.vi v9, v8, 1
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <2 x double> @llvm.vector.reverse.v2f64(<2 x double> %a)
+  %res = shufflevector <2 x double> %a, <2 x double> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x double> %res
 }
 
@@ -630,7 +620,7 @@ define <4 x double> @reverse_v4f64(<4 x double> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <4 x double> @llvm.vector.reverse.v4f64(<4 x double> %a)
+  %res = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x double> %res
 }
 
@@ -644,71 +634,31 @@ define <8 x double> @reverse_v8f64(<8 x double> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <8 x double> @llvm.vector.reverse.v8f64(<8 x double> %a)
+  %res = shufflevector <8 x double> %a, <8 x double> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x double> %res
 }
 
 
 define <3 x i64> @reverse_v3i64(<3 x i64> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v3i64:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    lui a0, %hi(.LCPI44_0)
-; RV32-BITS-UNKNOWN-NEXT:    addi a0, a0, %lo(.LCPI44_0)
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle16.v v12, (a0)
-; RV32-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV32-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v10
-; RV32-BITS-UNKNOWN-NEXT:    ret
+; RV32-LABEL: reverse_v3i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a0, %hi(.LCPI44_0)
+; RV32-NEXT:    addi a0, a0, %lo(.LCPI44_0)
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vle16.v v12, (a0)
+; RV32-NEXT:    vrgatherei16.vv v10, v8, v12
+; RV32-NEXT:    vmv.v.v v8, v10
+; RV32-NEXT:    ret
 ;
-; RV32-BITS-256-LABEL: reverse_v3i64:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    lui a0, %hi(.LCPI44_0)
-; RV32-BITS-256-NEXT:    addi a0, a0, %lo(.LCPI44_0)
-; RV32-BITS-256-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-BITS-256-NEXT:    vle16.v v12, (a0)
-; RV32-BITS-256-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV32-BITS-256-NEXT:    vmv.v.v v8, v10
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v3i64:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    lui a0, %hi(.LCPI44_0)
-; RV32-BITS-512-NEXT:    addi a0, a0, %lo(.LCPI44_0)
-; RV32-BITS-512-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-BITS-512-NEXT:    vle16.v v12, (a0)
-; RV32-BITS-512-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV32-BITS-512-NEXT:    vmv.v.v v8, v10
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v3i64:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vid.v v10
-; RV64-BITS-UNKNOWN-NEXT:    vrsub.vi v12, v10, 2
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV64-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v10
-; RV64-BITS-UNKNOWN-NEXT:    ret
-;
-; RV64-BITS-256-LABEL: reverse_v3i64:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64-BITS-256-NEXT:    vid.v v10
-; RV64-BITS-256-NEXT:    vrsub.vi v12, v10, 2
-; RV64-BITS-256-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-BITS-256-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV64-BITS-256-NEXT:    vmv.v.v v8, v10
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v3i64:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64-BITS-512-NEXT:    vid.v v10
-; RV64-BITS-512-NEXT:    vrsub.vi v12, v10, 2
-; RV64-BITS-512-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-BITS-512-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV64-BITS-512-NEXT:    vmv.v.v v8, v10
-; RV64-BITS-512-NEXT:    ret
+; RV64-LABEL: reverse_v3i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV64-NEXT:    vid.v v10
+; RV64-NEXT:    vrsub.vi v12, v10, 2
+; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v10, v8, v12
+; RV64-NEXT:    vmv.v.v v8, v10
+; RV64-NEXT:    ret
 ;
 ; RV32-ZVBB-LABEL: reverse_v3i64:
 ; RV32-ZVBB:       # %bb.0:
@@ -729,70 +679,30 @@ define <3 x i64> @reverse_v3i64(<3 x i64> %a) {
 ; RV64-ZVBB-NEXT:    vrgatherei16.vv v10, v8, v12
 ; RV64-ZVBB-NEXT:    vmv.v.v v8, v10
 ; RV64-ZVBB-NEXT:    ret
-  %res = call <3 x i64> @llvm.vector.reverse.v3i64(<3 x i64> %a)
+  %res = shufflevector <3 x i64> %a, <3 x i64> poison, <3 x i32> <i32 2, i32 1, i32 0>
   ret <3 x i64> %res
 }
 
 define <6 x i64> @reverse_v6i64(<6 x i64> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v6i64:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    lui a0, %hi(.LCPI45_0)
-; RV32-BITS-UNKNOWN-NEXT:    addi a0, a0, %lo(.LCPI45_0)
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle16.v v16, (a0)
-; RV32-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV32-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v12
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v6i64:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    lui a0, %hi(.LCPI45_0)
-; RV32-BITS-256-NEXT:    addi a0, a0, %lo(.LCPI45_0)
-; RV32-BITS-256-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-BITS-256-NEXT:    vle16.v v16, (a0)
-; RV32-BITS-256-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV32-BITS-256-NEXT:    vmv.v.v v8, v12
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v6i64:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    lui a0, %hi(.LCPI45_0)
-; RV32-BITS-512-NEXT:    addi a0, a0, %lo(.LCPI45_0)
-; RV32-BITS-512-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-BITS-512-NEXT:    vle16.v v16, (a0)
-; RV32-BITS-512-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV32-BITS-512-NEXT:    vmv.v.v v8, v12
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v6i64:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vid.v v12
-; RV64-BITS-UNKNOWN-NEXT:    vrsub.vi v16, v12, 5
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV64-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v12
-; RV64-BITS-UNKNOWN-NEXT:    ret
+; RV32-LABEL: reverse_v6i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a0, %hi(.LCPI45_0)
+; RV32-NEXT:    addi a0, a0, %lo(.LCPI45_0)
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle16.v v16, (a0)
+; RV32-NEXT:    vrgatherei16.vv v12, v8, v16
+; RV32-NEXT:    vmv.v.v v8, v12
+; RV32-NEXT:    ret
 ;
-; RV64-BITS-256-LABEL: reverse_v6i64:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-BITS-256-NEXT:    vid.v v12
-; RV64-BITS-256-NEXT:    vrsub.vi v16, v12, 5
-; RV64-BITS-256-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64-BITS-256-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV64-BITS-256-NEXT:    vmv.v.v v8, v12
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v6i64:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-BITS-512-NEXT:    vid.v v12
-; RV64-BITS-512-NEXT:    vrsub.vi v16, v12, 5
-; RV64-BITS-512-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64-BITS-512-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV64-BITS-512-NEXT:    vmv.v.v v8, v12
-; RV64-BITS-512-NEXT:    ret
+; RV64-LABEL: reverse_v6i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vid.v v12
+; RV64-NEXT:    vrsub.vi v16, v12, 5
+; RV64-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v12, v8, v16
+; RV64-NEXT:    vmv.v.v v8, v12
+; RV64-NEXT:    ret
 ;
 ; RV32-ZVBB-LABEL: reverse_v6i64:
 ; RV32-ZVBB:       # %bb.0:
@@ -813,73 +723,31 @@ define <6 x i64> @reverse_v6i64(<6 x i64> %a) {
 ; RV64-ZVBB-NEXT:    vrgatherei16.vv v12, v8, v16
 ; RV64-ZVBB-NEXT:    vmv.v.v v8, v12
 ; RV64-ZVBB-NEXT:    ret
-  %res = call <6 x i64> @llvm.vector.reverse.v6i64(<6 x i64> %a)
+  %res = shufflevector <6 x i64> %a, <6 x i64> poison, <6 x i32> <i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <6 x i64> %res
 }
 
 define <12 x i64> @reverse_v12i64(<12 x i64> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v12i64:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    li a0, 32
-; RV32-BITS-UNKNOWN-NEXT:    lui a1, %hi(.LCPI46_0)
-; RV32-BITS-UNKNOWN-NEXT:    addi a1, a1, %lo(.LCPI46_0)
-; RV32-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle16.v v24, (a1)
-; RV32-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV32-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v16
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v12i64:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    li a0, 32
-; RV32-BITS-256-NEXT:    lui a1, %hi(.LCPI46_0)
-; RV32-BITS-256-NEXT:    addi a1, a1, %lo(.LCPI46_0)
-; RV32-BITS-256-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-BITS-256-NEXT:    vle16.v v24, (a1)
-; RV32-BITS-256-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV32-BITS-256-NEXT:    vmv.v.v v8, v16
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v12i64:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    li a0, 32
-; RV32-BITS-512-NEXT:    lui a1, %hi(.LCPI46_0)
-; RV32-BITS-512-NEXT:    addi a1, a1, %lo(.LCPI46_0)
-; RV32-BITS-512-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-BITS-512-NEXT:    vle16.v v24, (a1)
-; RV32-BITS-512-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV32-BITS-512-NEXT:    vmv.v.v v8, v16
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v12i64:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vid.v v16
-; RV64-BITS-UNKNOWN-NEXT:    vrsub.vi v24, v16, 11
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV64-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v16
-; RV64-BITS-UNKNOWN-NEXT:    ret
+; RV32-LABEL: reverse_v12i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    lui a1, %hi(.LCPI46_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI46_0)
+; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; RV32-NEXT:    vle16.v v24, (a1)
+; RV32-NEXT:    vrgatherei16.vv v16, v8, v24
+; RV32-NEXT:    vmv.v.v v8, v16
+; RV32-NEXT:    ret
 ;
-; RV64-BITS-256-LABEL: reverse_v12i64:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-BITS-256-NEXT:    vid.v v16
-; RV64-BITS-256-NEXT:    vrsub.vi v24, v16, 11
-; RV64-BITS-256-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-BITS-256-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV64-BITS-256-NEXT:    vmv.v.v v8, v16
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v12i64:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-BITS-512-NEXT:    vid.v v16
-; RV64-BITS-512-NEXT:    vrsub.vi v24, v16, 11
-; RV64-BITS-512-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-BITS-512-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV64-BITS-512-NEXT:    vmv.v.v v8, v16
-; RV64-BITS-512-NEXT:    ret
+; RV64-LABEL: reverse_v12i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; RV64-NEXT:    vid.v v16
+; RV64-NEXT:    vrsub.vi v24, v16, 11
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v16, v8, v24
+; RV64-NEXT:    vmv.v.v v8, v16
+; RV64-NEXT:    ret
 ;
 ; RV32-ZVBB-LABEL: reverse_v12i64:
 ; RV32-ZVBB:       # %bb.0:
@@ -901,54 +769,484 @@ define <12 x i64> @reverse_v12i64(<12 x i64> %a) {
 ; RV64-ZVBB-NEXT:    vrgatherei16.vv v16, v8, v24
 ; RV64-ZVBB-NEXT:    vmv.v.v v8, v16
 ; RV64-ZVBB-NEXT:    ret
-  %res = call <12 x i64> @llvm.vector.reverse.v12i64(<12 x i64> %a)
+  %res = shufflevector <12 x i64> %a, <12 x i64> poison, <12 x i32> <i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <12 x i64> %res
 }
 
-declare <2 x i1> @llvm.vector.reverse.v2i1(<2 x i1>)
-declare <4 x i1> @llvm.vector.reverse.v4i1(<4 x i1>)
-declare <8 x i1> @llvm.vector.reverse.v8i1(<8 x i1>)
-declare <16 x i1> @llvm.vector.reverse.v16i1(<16 x i1>)
-declare <32 x i1> @llvm.vector.reverse.v32i1(<32 x i1>)
-declare <64 x i1> @llvm.vector.reverse.v64i1(<64 x i1>)
-declare <128 x i1> @llvm.vector.reverse.v128i1(<128 x i1>)
-declare <1 x i8> @llvm.vector.reverse.v1i8(<1 x i8>)
-declare <2 x i8> @llvm.vector.reverse.v2i8(<2 x i8>)
-declare <4 x i8> @llvm.vector.reverse.v4i8(<4 x i8>)
-declare <8 x i8> @llvm.vector.reverse.v8i8(<8 x i8>)
-declare <16 x i8> @llvm.vector.reverse.v16i8(<16 x i8>)
-declare <32 x i8> @llvm.vector.reverse.v32i8(<32 x i8>)
-declare <64 x i8> @llvm.vector.reverse.v64i8(<64 x i8>)
-declare <1 x i16> @llvm.vector.reverse.v1i16(<1 x i16>)
-declare <2 x i16> @llvm.vector.reverse.v2i16(<2 x i16>)
-declare <4 x i16> @llvm.vector.reverse.v4i16(<4 x i16>)
-declare <8 x i16> @llvm.vector.reverse.v8i16(<8 x i16>)
-declare <16 x i16> @llvm.vector.reverse.v16i16(<16 x i16>)
-declare <32 x i16> @llvm.vector.reverse.v32i16(<32 x i16>)
-declare <1 x i32> @llvm.vector.reverse.v1i32(<1 x i32>)
-declare <2 x i32> @llvm.vector.reverse.v2i32(<2 x i32>)
-declare <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32>)
-declare <8 x i32> @llvm.vector.reverse.v8i32(<8 x i32>)
-declare <16 x i32> @llvm.vector.reverse.v16i32(<16 x i32>)
-declare <1 x i64> @llvm.vector.reverse.v1i64(<1 x i64>)
-declare <2 x i64> @llvm.vector.reverse.v2i64(<2 x i64>)
-declare <4 x i64> @llvm.vector.reverse.v4i64(<4 x i64>)
-declare <8 x i64> @llvm.vector.reverse.v8i64(<8 x i64>)
-declare <1 x half> @llvm.vector.reverse.v1f16(<1 x half>)
-declare <2 x half> @llvm.vector.reverse.v2f16(<2 x half>)
-declare <4 x half> @llvm.vector.reverse.v4f16(<4 x half>)
-declare <8 x half> @llvm.vector.reverse.v8f16(<8 x half>)
-declare <16 x half> @llvm.vector.reverse.v16f16(<16 x half>)
-declare <32 x half> @llvm.vector.reverse.v32f16(<32 x half>)
-declare <1 x float> @llvm.vector.reverse.v1f32(<1 x float>)
-declare <2 x float> @llvm.vector.reverse.v2f32(<2 x float>)
-declare <4 x float> @llvm.vector.reverse.v4f32(<4 x float>)
-declare <8 x float> @llvm.vector.reverse.v8f32(<8 x float>)
-declare <16 x float> @llvm.vector.reverse.v16f32(<16 x float>)
-declare <1 x double> @llvm.vector.reverse.v1f64(<1 x double>)
-declare <2 x double> @llvm.vector.reverse.v2f64(<2 x double>)
-declare <4 x double> @llvm.vector.reverse.v4f64(<4 x double>)
-declare <8 x double> @llvm.vector.reverse.v8f64(<8 x double>)
-declare <3 x i64> @llvm.vector.reverse.v3i64(<3 x i64>)
-declare <6 x i64> @llvm.vector.reverse.v6i64(<6 x i64>)
-declare <12 x i64> @llvm.vector.reverse.v12i64(<12 x i64>)
+define <4 x i8> @reverse_v4i8_2(<2 x i8> %a, <2 x i8> %b) {
+; NO-ZVBB-LABEL: reverse_v4i8_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4i8_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZVBB-NEXT:    vrev8.v v10, v8
+; ZVBB-NEXT:    vrev8.v v8, v9
+; ZVBB-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i8> %res
+}
+
+define <8 x i8> @reverse_v8i8_2(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: reverse_v8i8_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    vrsub.vi v12, v11, 7
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vrsub.vi v8, v11, 3
+; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %res
+}
+
+define <16 x i8> @reverse_v16i8_2(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: reverse_v16i8_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    vrsub.vi v12, v11, 15
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
+; CHECK-NEXT:    vrsub.vi v8, v11, 7
+; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i8> %res
+}
+
+define <32 x i8> @reverse_v32i8_2(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: reverse_v32i8_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vid.v v14
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v14, a0
+; CHECK-NEXT:    vrgather.vv v10, v8, v16
+; CHECK-NEXT:    vrsub.vi v8, v14, 15
+; CHECK-NEXT:    lui a0, 16
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, mu
+; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <16 x i8> %a, <16 x i8> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <32 x i8> %res
+}
+
+define <4 x i16> @reverse_v4i16_2(<2 x i16> %a, <2 x i16> %b) {
+; NO-ZVBB-LABEL: reverse_v4i16_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4i16_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; ZVBB-NEXT:    vror.vi v10, v8, 16
+; ZVBB-NEXT:    vror.vi v8, v9, 16
+; ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x i16> %a, <2 x i16> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @reverse_v8i16_2(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: reverse_v8i16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    vrsub.vi v12, v11, 7
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vrsub.vi v8, v11, 3
+; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i16> %res
+}
+
+define <16 x i16> @reverse_v16i16_2(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: reverse_v16i16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
+; CHECK-NEXT:    vid.v v14
+; CHECK-NEXT:    vrsub.vi v16, v14, 15
+; CHECK-NEXT:    vrgather.vv v10, v8, v16
+; CHECK-NEXT:    vrsub.vi v8, v14, 7
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x i16> %a, <8 x i16> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i16> %res
+}
+
+define <32 x i16> @reverse_v32i16_2(<16 x i16> %a, <16 x i16> %b) {
+; CHECK-LABEL: reverse_v32i16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v20
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v24, v20, a0
+; CHECK-NEXT:    vrgather.vv v12, v8, v24
+; CHECK-NEXT:    vrsub.vi v8, v20, 15
+; CHECK-NEXT:    lui a0, 16
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vrgather.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <16 x i16> %a, <16 x i16> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <32 x i16> %res
+}
+
+define <4 x i32> @reverse_v4i32_2(<2 x i32> %a, < 2 x i32> %b) {
+; NO-ZVBB-LABEL: reverse_v4i32_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4i32_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; ZVBB-NEXT:    vror.vi v10, v8, 32
+; ZVBB-NEXT:    vror.vi v8, v9, 32
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @reverse_v8i32_2(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: reverse_v8i32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vi v13, v9, 7
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v13
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v9, 3
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i32> %res
+}
+
+define <16 x i32> @reverse_v16i32_2(<8 x i32> %a, <8 x i32> %b) {
+; CHECK-LABEL: reverse_v16i32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v18, v10, 15
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v18
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 7
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x i32> %a, <8 x i32> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i32> %res
+}
+
+define <32 x i32> @reverse_v32i32_2(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: reverse_v32i32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv4r.v v24, v12
+; CHECK-NEXT:    vmv4r.v v16, v8
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v20
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v28, v20, a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v8, v16, v28
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v16, v20, 15
+; CHECK-NEXT:    lui a0, 16
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vrgatherei16.vv v8, v24, v16, v0.t
+; CHECK-NEXT:    ret
+  %res = shufflevector <16 x i32> %a, <16 x i32> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <32 x i32> %res
+}
+
+define <4 x i64> @reverse_v4i64_2(<2 x i64> %a, < 2 x i64> %b) {
+; CHECK-LABEL: reverse_v4i64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 1
+; CHECK-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-NEXT:    vslidedown.vi v8, v9, 1
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-NEXT:    ret
+  %res = shufflevector <2 x i64> %a, <2 x i64> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i64> %res
+}
+
+define <8 x i64> @reverse_v8i64_2(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: reverse_v8i64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v11, v10, 7
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v11
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 3
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x i64> %a, <4 x i64> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i64> %res
+}
+
+define <4 x half> @reverse_v4f16_2(<2 x half> %a, <2 x half> %b) {
+; NO-ZVBB-LABEL: reverse_v4f16_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4f16_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; ZVBB-NEXT:    vror.vi v10, v8, 16
+; ZVBB-NEXT:    vror.vi v8, v9, 16
+; ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x half> %a, <2 x half> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x half> %res
+}
+
+define <8 x half> @reverse_v8f16_2(<4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: reverse_v8f16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    vrsub.vi v12, v11, 7
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vrsub.vi v8, v11, 3
+; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x half> %a, <4 x half> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %res
+}
+
+define <16 x half> @reverse_v16f16_2(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: reverse_v16f16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
+; CHECK-NEXT:    vid.v v14
+; CHECK-NEXT:    vrsub.vi v16, v14, 15
+; CHECK-NEXT:    vrgather.vv v10, v8, v16
+; CHECK-NEXT:    vrsub.vi v8, v14, 7
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x half> %a, <8 x half> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x half> %res
+}
+
+define <32 x half> @reverse_v32f16_2(<16 x half> %a) {
+; CHECK-LABEL: reverse_v32f16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
+; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <16 x half> %a, <16 x half> poison,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <32 x half> %res
+}
+
+define <4 x float> @reverse_v4f32_2(<2 x float> %a, <2 x float> %b) {
+; NO-ZVBB-LABEL: reverse_v4f32_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4f32_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; ZVBB-NEXT:    vror.vi v10, v8, 32
+; ZVBB-NEXT:    vror.vi v8, v9, 32
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x float> %res
+}
+
+define <8 x float> @reverse_v8f32_2(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: reverse_v8f32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vi v13, v9, 7
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v13
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v9, 3
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x float> %a, <4 x float> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x float> %res
+}
+
+define <16 x float> @reverse_v16f32_2(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: reverse_v16f32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v18, v10, 15
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v18
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 7
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x float> %a, <8 x float> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x float> %res
+}
+
+define <4 x double> @reverse_v4f64_2(<2 x double> %a, < 2 x double> %b) {
+; CHECK-LABEL: reverse_v4f64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 1
+; CHECK-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-NEXT:    vslidedown.vi v8, v9, 1
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-NEXT:    ret
+  %res = shufflevector <2 x double> %a, <2 x double> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x double> %res
+}
+
+define <8 x double> @reverse_v8f64_2(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: reverse_v8f64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v11, v10, 7
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v11
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 3
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x double> %a, <4 x double> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x double> %res
+}
+
+; There is no corresponding v1i256 type, so make sure we don't crash if we try
+; to lower via lowerBitreverseShuffle.
+define <256 x i1> @reverse_v256i1(<256 x i1> %a) vscale_range(16, 1024) {
+; CHECK-LABEL: reverse_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 256
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, -1
+; CHECK-NEXT:    vrgather.vv v12, v8, v10
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    ret
+  %res = shufflevector <256 x i1> %a, <256 x i1> poison, <256 x i32> <i32 255, i32 254, i32 253, i32 252, i32 251, i32 250, i32 249, i32 248, i32 247, i32 246, i32 245, i32 244, i32 243, i32 242, i32 241, i32 240, i32 239, i32 238, i32 237, i32 236, i32 235, i32 234, i32 233, i32 232, i32 231, i32 230, i32 229, i32 228, i32 227, i32 226, i32 225, i32 224, i32 223, i32 222, i32 221, i32 220, i32 219, i32 218, i32 217, i32 216, i32 215, i32 214, i32 213, i32 212, i32 211, i32 210, i32 209, i32 208, i32 207, i32 206, i32 205, i32 204, i32 203, i32 202, i32 201, i32 200, i32 199, i32 198, i32 197, i32 196, i32 195, i32 194, i32 193, i32 192, i32 191, i32 190, i32 189, i32 188, i32 187, i32 186, i32 185, i32 184, i32 183, i32 182, i32 181, i32 180, i32 179, i32 178, i32 177, i32 176, i32 175, i32 174, i32 173, i32 172, i32 171, i32 170, i32 169, i32 168, i32 167, i32 166, i32 165, i32 164, i32 163, i32 162, i32 161, i32 160, i32 159, i32 158, i32 157, i32 156, i32 155, i32 154, i32 153, i32 152, i32 151, i32 150, i32 149, i32 148, i32 147, i32 146, i32 145, i32 144, i32 143, i32 142, i32 141, i32 140, i32 139, i32 138, i32 137, i32 136, i32 135, i32 134, i32 133, i32 132, i32 131, i32 130, i32 129, i32 128, i32 127, i32 126, i32 125, i32 124, i32 123, i32 122, i32 121, i32 120, i32 119, i32 118, i32 117, i32 116, i32 115, i32 114, i32 113, i32 112, i32 111, i32 110, i32 109, i32 108, i32 107, i32 106, i32 105, i32 104, i32 103, i32 102, i32 101, i32 100, i32 99, i32 98, i32 97, i32 96, i32 95, i32 94, i32 93, i32 92, i32 91, i32 90, i32 89, i32 88, i32 87, i32 86, i32 85, i32 84, i32 83, i32 82, i32 81, i32 80, i32 79, i32 78, i32 77, i32 76, i32 75, i32 74, i32 73, i32 72, i32 71, i32 70, i32 69, i32 68, i32 67, i32 66, i32 65, i32 64, i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <256 x i1> %res
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
index e2f540e991fd0b1..4e367bb0d70cd1c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
@@ -101,8 +101,10 @@ define void @trunc_sat_u8u16_notopt(ptr %x, ptr %y) {
 define void @trunc_sat_u8u16_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u16_maxmin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -117,8 +119,10 @@ define void @trunc_sat_u8u16_maxmin(ptr %x, ptr %y) {
 define void @trunc_sat_u8u16_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u16_minmax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -352,8 +356,10 @@ define void @trunc_sat_u32u64_min(ptr %x, ptr %y) {
 define void @trunc_sat_u32u64_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u32u64_maxmin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vse32.v v10, (a1)
 ; CHECK-NEXT:    ret
@@ -368,8 +374,10 @@ define void @trunc_sat_u32u64_maxmin(ptr %x, ptr %y) {
 define void @trunc_sat_u32u64_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u32u64_minmax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vse32.v v10, (a1)
 ; CHECK-NEXT:    ret
@@ -437,8 +445,10 @@ define void @trunc_sat_u8u32_min(ptr %x, ptr %y) {
 define void @trunc_sat_u8u32_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u32_maxmin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
@@ -455,8 +465,10 @@ define void @trunc_sat_u8u32_maxmin(ptr %x, ptr %y) {
 define void @trunc_sat_u8u32_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u32_minmax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
@@ -532,8 +544,10 @@ define void @trunc_sat_u8u64_min(ptr %x, ptr %y) {
 define void @trunc_sat_u8u64_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u64_maxmin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -552,8 +566,10 @@ define void @trunc_sat_u8u64_maxmin(ptr %x, ptr %y) {
 define void @trunc_sat_u8u64_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u64_minmax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index 102ca143dfc737e..1395dc914bb4021 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -113,6 +113,7 @@ define <2 x i32> @ustest_f64i32(<2 x double> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; CHECK-V-NEXT:    vmax.vx v8, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
@@ -303,6 +304,9 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vfwcvt.rtz.x.f.v v10, v8
+; CHECK-V-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-V-NEXT:    vmax.vx v10, v10, zero
+; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    ret
 entry:
@@ -797,16 +801,17 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-V-NEXT:    vmv.s.x v10, a0
+; CHECK-V-NEXT:    vmv.s.x v8, a0
 ; CHECK-V-NEXT:    addi a0, sp, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
+; CHECK-V-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-V-NEXT:    vmax.vx v10, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    csrr a0, vlenb
@@ -939,8 +944,9 @@ define <2 x i16> @ustest_f64i16(<2 x double> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; CHECK-V-NEXT:    vmax.vx v8, v9, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-V-NEXT:    vnclipu.wi v8, v9, 0
+; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i32>
@@ -1133,6 +1139,7 @@ define <4 x i16> @ustest_f32i16(<4 x float> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; CHECK-V-NEXT:    vmax.vx v8, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
@@ -2107,23 +2114,24 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) {
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-V-NEXT:    vmv.s.x v10, a0
+; CHECK-V-NEXT:    vmv.s.x v8, a0
 ; CHECK-V-NEXT:    addi a0, sp, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 2
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    slli a0, a0, 1
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 4
+; CHECK-V-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-V-NEXT:    vmax.vx v10, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    csrr a0, vlenb
@@ -3465,6 +3473,7 @@ define <2 x i32> @ustest_f64i32_mm(<2 x double> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; CHECK-V-NEXT:    vmax.vx v8, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
@@ -3650,6 +3659,9 @@ define <4 x i32> @ustest_f32i32_mm(<4 x float> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vfwcvt.rtz.x.f.v v10, v8
+; CHECK-V-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-V-NEXT:    vmax.vx v10, v10, zero
+; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    ret
 entry:
@@ -4139,16 +4151,17 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-V-NEXT:    vmv.s.x v10, a0
+; CHECK-V-NEXT:    vmv.s.x v8, a0
 ; CHECK-V-NEXT:    addi a0, sp, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
+; CHECK-V-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-V-NEXT:    vmax.vx v10, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    csrr a0, vlenb
@@ -4276,8 +4289,9 @@ define <2 x i16> @ustest_f64i16_mm(<2 x double> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; CHECK-V-NEXT:    vmax.vx v8, v9, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-V-NEXT:    vnclipu.wi v8, v9, 0
+; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i32>
@@ -4465,6 +4479,7 @@ define <4 x i16> @ustest_f32i16_mm(<4 x float> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; CHECK-V-NEXT:    vmax.vx v8, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
@@ -5434,23 +5449,24 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) {
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-V-NEXT:    vmv.s.x v10, a0
+; CHECK-V-NEXT:    vmv.s.x v8, a0
 ; CHECK-V-NEXT:    addi a0, sp, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 2
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    slli a0, a0, 1
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 4
+; CHECK-V-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-V-NEXT:    vmax.vx v10, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    csrr a0, vlenb
diff --git a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
index 96094eea631ba50..60a03f1d97e5380 100644
--- a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
@@ -1285,10 +1285,11 @@ define <vscale x 4 x i32> @reverse_nxv4i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %a)
@@ -1300,10 +1301,11 @@ define <vscale x 8 x i32> @reverse_nxv8i32(<vscale x 8 x i32> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> %a)
@@ -1316,10 +1318,11 @@ define <vscale x 16 x i32> @reverse_nxv16i32(<vscale x 16 x i32> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v24, v16, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v24
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x i32> @llvm.vector.reverse.nxv16i32(<vscale x 16 x i32> %a)
@@ -1348,10 +1351,11 @@ define <vscale x 2 x i64> @reverse_nxv2i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> %a)
@@ -1364,10 +1368,11 @@ define <vscale x 4 x i64> @reverse_nxv4i64(<vscale x 4 x i64> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> %a)
@@ -1379,10 +1384,11 @@ define <vscale x 8 x i64> @reverse_nxv8i64(<vscale x 8 x i64> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v24, v16, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v24
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> %a)
@@ -1526,10 +1532,11 @@ define <vscale x 4 x float> @reverse_nxv4f32(<vscale x 4 x float> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
@@ -1541,10 +1548,11 @@ define <vscale x 8 x float> @reverse_nxv8f32(<vscale x 8 x float> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> %a)
@@ -1557,10 +1565,11 @@ define <vscale x 16 x float> @reverse_nxv16f32(<vscale x 16 x float> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v24, v16, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v24
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x float> @llvm.vector.reverse.nxv16f32(<vscale x 16 x float> %a)
@@ -1589,10 +1598,11 @@ define <vscale x 2 x double> @reverse_nxv2f64(<vscale x 2 x double> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %a)
@@ -1605,10 +1615,11 @@ define <vscale x 4 x double> @reverse_nxv4f64(<vscale x 4 x double> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> %a)
@@ -1620,10 +1631,11 @@ define <vscale x 8 x double> @reverse_nxv8f64(<vscale x 8 x double> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v24, v16, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v24
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x double> @llvm.vector.reverse.nxv8f64(<vscale x 8 x double> %a)
@@ -1638,10 +1650,11 @@ define <vscale x 3 x i64> @reverse_nxv3i64(<vscale x 3 x i64> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v12, v12, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v12
 ; CHECK-NEXT:    vmv1r.v v8, v17
 ; CHECK-NEXT:    vmv1r.v v9, v18
 ; CHECK-NEXT:    vmv1r.v v10, v19
@@ -1655,10 +1668,11 @@ define <vscale x 6 x i64> @reverse_nxv6i64(<vscale x 6 x i64> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v16, v16, a0
-; CHECK-NEXT:    vrgather.vv v24, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v24, v8, v16
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    vmv2r.v v10, v28
 ; CHECK-NEXT:    vmv2r.v v12, v30
@@ -1684,12 +1698,13 @@ define <vscale x 12 x i64> @reverse_nxv12i64(<vscale x 12 x i64> %a) {
 ; RV32-NEXT:    andi sp, sp, -64
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    addi a1, a0, -1
-; RV32-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
 ; RV32-NEXT:    vid.v v24
 ; RV32-NEXT:    vrsub.vx v24, v24, a1
-; RV32-NEXT:    vrgather.vv v0, v16, v24
+; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV32-NEXT:    vrgatherei16.vv v0, v16, v24
 ; RV32-NEXT:    vmv4r.v v16, v4
-; RV32-NEXT:    vrgather.vv v0, v8, v24
+; RV32-NEXT:    vrgatherei16.vv v0, v8, v24
 ; RV32-NEXT:    vmv4r.v v20, v0
 ; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    addi a1, sp, 64
@@ -1720,12 +1735,13 @@ define <vscale x 12 x i64> @reverse_nxv12i64(<vscale x 12 x i64> %a) {
 ; RV64-NEXT:    andi sp, sp, -64
 ; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    addi a1, a0, -1
-; RV64-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
 ; RV64-NEXT:    vid.v v24
 ; RV64-NEXT:    vrsub.vx v24, v24, a1
-; RV64-NEXT:    vrgather.vv v0, v16, v24
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v0, v16, v24
 ; RV64-NEXT:    vmv4r.v v16, v4
-; RV64-NEXT:    vrgather.vv v0, v8, v24
+; RV64-NEXT:    vrgatherei16.vv v0, v8, v24
 ; RV64-NEXT:    vmv4r.v v20, v0
 ; RV64-NEXT:    slli a0, a0, 3
 ; RV64-NEXT:    addi a1, sp, 64
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr104480.ll b/llvm/test/CodeGen/RISCV/rvv/pr104480.ll
new file mode 100644
index 000000000000000..93cf4d3766089a1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/pr104480.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v | FileCheck %s
+
+define <vscale x 4 x i16> @test_mulhs_promote(<vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) {
+; CHECK-LABEL: test_mulhs_promote:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrgather.vi v9, v8, 0
+; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    addi a0, a0, 1366
+; CHECK-NEXT:    vmulh.vx v8, v9, a0
+; CHECK-NEXT:    vsrl.vi v10, v8, 15
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vnmsub.vx v8, a0, v9
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splat = shufflevector <vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i16> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %2 = srem <vscale x 4 x i16> %broadcast.splat, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 3, i64 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i16> %2
+}
+
+define <vscale x 4 x i16> @test_mulhu_promote(<vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) {
+; CHECK-LABEL: test_mulhu_promote:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrgather.vi v9, v8, 0
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, -1365
+; CHECK-NEXT:    vmulhu.vx v8, v9, a0
+; CHECK-NEXT:    vsrl.vi v8, v8, 1
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vnmsub.vx v8, a0, v9
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splat = shufflevector <vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i16> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %2 = urem <vscale x 4 x i16> %broadcast.splat, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 3, i64 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i16> %2
+}
+
+define <vscale x 4 x i64> @test_mulhs_expand(<vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) {
+; CHECK-LABEL: test_mulhs_expand:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    addi a1, a0, 1365
+; CHECK-NEXT:    sw a1, 12(sp)
+; CHECK-NEXT:    addi a0, a0, 1366
+; CHECK-NEXT:    sw a0, 8(sp)
+; CHECK-NEXT:    addi a0, sp, 8
+; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vlse64.v v12, (a0), zero
+; CHECK-NEXT:    vrgather.vi v16, v8, 0
+; CHECK-NEXT:    vmulh.vv v8, v16, v12
+; CHECK-NEXT:    li a0, 63
+; CHECK-NEXT:    vsrl.vx v12, v8, a0
+; CHECK-NEXT:    vadd.vv v8, v8, v12
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vnmsub.vx v8, a0, v16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splat = shufflevector <vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i64> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %2 = srem <vscale x 4 x i64> %broadcast.splat, shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @test_mulhu_expand(<vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) {
+; CHECK-LABEL: test_mulhu_expand:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    lui a0, 699051
+; CHECK-NEXT:    addi a1, a0, -1366
+; CHECK-NEXT:    sw a1, 12(sp)
+; CHECK-NEXT:    addi a0, a0, -1365
+; CHECK-NEXT:    sw a0, 8(sp)
+; CHECK-NEXT:    addi a0, sp, 8
+; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vlse64.v v12, (a0), zero
+; CHECK-NEXT:    vrgather.vi v16, v8, 0
+; CHECK-NEXT:    vmulhu.vv v8, v16, v12
+; CHECK-NEXT:    vsrl.vi v8, v8, 1
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vnmsub.vx v8, a0, v16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splat = shufflevector <vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i64> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %2 = urem <vscale x 4 x i64> %broadcast.splat, shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i64> %2
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
deleted file mode 100644
index 368f454fa5fda1f..000000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
+++ /dev/null
@@ -1,736 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+f,+zfh,+zvfh,+d -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+f,+zfh,+zvfh,+d -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,RV64
-
-define <2 x i8> @v2i8(<2 x i8> %a) {
-; CHECK-LABEL: v2i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2i8 = shufflevector <2 x i8> %a, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x i8> %v2i8
-}
-
-define <4 x i8> @v2i8_2(<2 x i8> %a, <2 x i8> %b) {
-; CHECK-LABEL: v2i8_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4i8 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i8> %v4i8
-}
-
-define <4 x i8> @v4i8(<4 x i8> %a) {
-; CHECK-LABEL: v4i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v4i8 = shufflevector <4 x i8> %a, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i8> %v4i8
-}
-
-define <8 x i8> @v4i8_2(<4 x i8> %a, <4 x i8> %b) {
-; CHECK-LABEL: v4i8_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv1r.v v8, v10
-; CHECK-NEXT:    ret
-  %v8i8 = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i8> %v8i8
-}
-
-define <8 x i8> @v8i8(<8 x i8> %a) {
-; CHECK-LABEL: v8i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 7
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v8i8 = shufflevector <8 x i8> %a, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i8> %v8i8
-}
-
-define <16 x i8> @v8i8_2(<8 x i8> %a, <8 x i8> %b) {
-; CHECK-LABEL: v8i8_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
-; CHECK-NEXT:    vrsub.vi v8, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16i8 = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i8> %v16i8
-}
-
-define <16 x i8> @v16i8(<16 x i8> %a) {
-; CHECK-LABEL: v16i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 15
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v16i8 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i8> %v16i8
-}
-
-define <32 x i8> @v16i8_2(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: v16i8_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI7_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI7_0)
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
-; CHECK-NEXT:    vmv1r.v v14, v9
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    vrsub.vi v8, v8, 15
-; CHECK-NEXT:    lui a0, 16
-; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, mu
-; CHECK-NEXT:    vrgather.vv v10, v14, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v32i8 = shufflevector <16 x i8> %a, <16 x i8> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x i8> %v32i8
-}
-
-define <2 x i16> @v2i16(<2 x i16> %a) {
-; CHECK-LABEL: v2i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2i16 = shufflevector <2 x i16> %a, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x i16> %v2i16
-}
-
-define <4 x i16> @v2i16_2(<2 x i16> %a, <2 x i16> %b) {
-; CHECK-LABEL: v2i16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4i16 = shufflevector <2 x i16> %a, <2 x i16> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i16> %v4i16
-}
-
-define <4 x i16> @v4i16(<4 x i16> %a) {
-; CHECK-LABEL: v4i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v4i16 = shufflevector <4 x i16> %a, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i16> %v4i16
-}
-
-define <8 x i16> @v4i16_2(<4 x i16> %a, <4 x i16> %b) {
-; CHECK-LABEL: v4i16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8i16 = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i16> %v8i16
-}
-
-define <8 x i16> @v8i16(<8 x i16> %a) {
-; CHECK-LABEL: v8i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 7
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v8i16 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i16> %v8i16
-}
-
-define <16 x i16> @v8i16_2(<8 x i16> %a, <8 x i16> %b) {
-; CHECK-LABEL: v8i16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
-; CHECK-NEXT:    vid.v v14
-; CHECK-NEXT:    vrsub.vi v16, v14, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v16
-; CHECK-NEXT:    vrsub.vi v8, v14, 7
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16i16 = shufflevector <8 x i16> %a, <8 x i16> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i16> %v16i16
-}
-
-define <16 x i16> @v16i16(<16 x i16> %a) {
-; CHECK-LABEL: v16i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16i16 = shufflevector <16 x i16> %a, <16 x i16> undef,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i16> %v16i16
-}
-
-define <32 x i16> @v16i16_2(<16 x i16> %a, <16 x i16> %b) {
-; CHECK-LABEL: v16i16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI15_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI15_0)
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vle16.v v16, (a1)
-; CHECK-NEXT:    vmv2r.v v20, v10
-; CHECK-NEXT:    vmv2r.v v12, v8
-; CHECK-NEXT:    vrgather.vv v8, v12, v16
-; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    vrsub.vi v12, v12, 15
-; CHECK-NEXT:    lui a0, 16
-; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
-; CHECK-NEXT:    vrgather.vv v8, v20, v12, v0.t
-; CHECK-NEXT:    ret
-  %v32i16 = shufflevector <16 x i16> %a, <16 x i16> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x i16> %v32i16
-}
-
-define <2 x i32> @v2i32(<2 x i32> %a) {
-; CHECK-LABEL: v2i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2i32 = shufflevector <2 x i32> %a, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x i32> %v2i32
-}
-
-define <4 x i32> @v2i32_2(<2 x i32> %a, < 2 x i32> %b) {
-; CHECK-LABEL: v2i32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4i32 = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i32> %v4i32
-}
-
-define <4 x i32> @v4i32(<4 x i32> %a) {
-; CHECK-LABEL: v4i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v4i32 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i32> %v4i32
-}
-
-define <8 x i32> @v4i32_2(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-LABEL: v4i32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v13, v9, 7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v13
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v9, 3
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8i32 = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i32> %v8i32
-}
-
-define <8 x i32> @v8i32(<8 x i32> %a) {
-; CHECK-LABEL: v8i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8i32 = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i32> %v8i32
-}
-
-define <16 x i32> @v8i32_2(<8 x i32> %a, <8 x i32> %b) {
-; CHECK-LABEL: v8i32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v18, v10, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v18
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 7
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v16i32 = shufflevector <8 x i32> %a, <8 x i32> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i32> %v16i32
-}
-
-define <16 x i32> @v16i32(<16 x i32> %a) {
-; CHECK-LABEL: v16i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    vrsub.vi v16, v12, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v16i32 = shufflevector <16 x i32> %a, <16 x i32> undef,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i32> %v16i32
-}
-
-define <32 x i32> @v16i32_2(<16 x i32> %a, <16 x i32> %b) {
-; CHECK-LABEL: v16i32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI23_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI23_0)
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    vle16.v v20, (a1)
-; CHECK-NEXT:    vmv4r.v v24, v12
-; CHECK-NEXT:    vmv4r.v v16, v8
-; CHECK-NEXT:    vrgatherei16.vv v8, v16, v20
-; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vid.v v16
-; CHECK-NEXT:    vrsub.vi v16, v16, 15
-; CHECK-NEXT:    lui a0, 16
-; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v24, v16, v0.t
-; CHECK-NEXT:    ret
-  %v32i32 = shufflevector <16 x i32> %a, <16 x i32> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x i32> %v32i32
-}
-
-define <2 x i64> @v2i64(<2 x i64> %a) {
-; CHECK-LABEL: v2i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v2i64 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x i64> %v2i64
-}
-
-define <4 x i64> @v2i64_2(<2 x i64> %a, < 2 x i64> %b) {
-; CHECK-LABEL: v2i64_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4i64 = shufflevector <2 x i64> %a, <2 x i64> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i64> %v4i64
-}
-
-define <4 x i64> @v4i64(<4 x i64> %a) {
-; CHECK-LABEL: v4i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 3
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v4i64 = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i64> %v4i64
-}
-
-define <8 x i64> @v4i64_2(<4 x i64> %a, <4 x i64> %b) {
-; CHECK-LABEL: v4i64_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v11, v10, 7
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v11
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 3
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v8i64 = shufflevector <4 x i64> %a, <4 x i64> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i64> %v8i64
-}
-
-define <2 x half> @v2f16(<2 x half> %a) {
-; CHECK-LABEL: v2f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2f16 = shufflevector <2 x half> %a, <2 x half> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x half> %v2f16
-}
-
-define <4 x half> @v2f16_2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: v2f16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4f16 = shufflevector <2 x half> %a, <2 x half> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x half> %v4f16
-}
-
-define <4 x half> @v4f16(<4 x half> %a) {
-; CHECK-LABEL: v4f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v4f16 = shufflevector <4 x half> %a, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x half> %v4f16
-}
-
-define <8 x half> @v4f16_2(<4 x half> %a, <4 x half> %b) {
-; CHECK-LABEL: v4f16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8f16 = shufflevector <4 x half> %a, <4 x half> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x half> %v8f16
-}
-
-define <8 x half> @v8f16(<8 x half> %a) {
-; CHECK-LABEL: v8f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 7
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v8f16 = shufflevector <8 x half> %a, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x half> %v8f16
-}
-
-define <16 x half> @v8f16_2(<8 x half> %a, <8 x half> %b) {
-; CHECK-LABEL: v8f16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
-; CHECK-NEXT:    vid.v v14
-; CHECK-NEXT:    vrsub.vi v16, v14, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v16
-; CHECK-NEXT:    vrsub.vi v8, v14, 7
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16f16 = shufflevector <8 x half> %a, <8 x half> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x half> %v16f16
-}
-
-define <16 x half> @v16f16(<16 x half> %a) {
-; CHECK-LABEL: v16f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16f16 = shufflevector <16 x half> %a, <16 x half> undef,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x half> %v16f16
-}
-
-define <32 x half> @v16f16_2(<16 x half> %a) {
-; CHECK-LABEL: v16f16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI35_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI35_0)
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vle16.v v16, (a1)
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v32f16 = shufflevector <16 x half> %a, <16 x half> undef,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x half> %v32f16
-}
-
-define <2 x float> @v2f32(<2 x float> %a) {
-; CHECK-LABEL: v2f32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2f32 = shufflevector <2 x float> %a, <2 x float> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x float> %v2f32
-}
-
-define <4 x float> @v2f32_2(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: v2f32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4f32 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x float> %v4f32
-}
-
-define <4 x float> @v4f32(<4 x float> %a) {
-; CHECK-LABEL: v4f32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v4f32 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x float> %v4f32
-}
-
-define <8 x float> @v4f32_2(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: v4f32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v13, v9, 7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v13
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v9, 3
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8f32 = shufflevector <4 x float> %a, <4 x float> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x float> %v8f32
-}
-
-define <8 x float> @v8f32(<8 x float> %a) {
-; CHECK-LABEL: v8f32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8f32 = shufflevector <8 x float> %a, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x float> %v8f32
-}
-
-define <16 x float> @v8f32_2(<8 x float> %a, <8 x float> %b) {
-; CHECK-LABEL: v8f32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v18, v10, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v18
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 7
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v16f32 = shufflevector <8 x float> %a, <8 x float> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x float> %v16f32
-}
-
-define <2 x double> @v2f64(<2 x double> %a) {
-; CHECK-LABEL: v2f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v2f64 = shufflevector <2 x double> %a, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x double> %v2f64
-}
-
-define <4 x double> @v2f64_2(<2 x double> %a, < 2 x double> %b) {
-; CHECK-LABEL: v2f64_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4f64 = shufflevector <2 x double> %a, <2 x double> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x double> %v4f64
-}
-
-define <4 x double> @v4f64(<4 x double> %a) {
-; CHECK-LABEL: v4f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 3
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v4f64 = shufflevector <4 x double> %a, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x double> %v4f64
-}
-
-define <8 x double> @v4f64_2(<4 x double> %a, <4 x double> %b) {
-; CHECK-LABEL: v4f64_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v11, v10, 7
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v11
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 3
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v8f64 = shufflevector <4 x double> %a, <4 x double> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x double> %v8f64
-}
-
-define <32 x i8> @v32i8(<32 x i8> %a) {
-; CHECK-LABEL: v32i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI46_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI46_0)
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v32i8 = shufflevector <32 x i8> %a, <32 x i8> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x i8> %v32i8
-}
-
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RV32: {{.*}}
-; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
index f43faadc532f260..01a90d8a33b6ec2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
@@ -102,7 +102,9 @@ define void @trunc_sat_u8u16_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u16_maxmin:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl1re16.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -118,7 +120,9 @@ define void @trunc_sat_u8u16_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u16_minmax:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl1re16.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -353,7 +357,9 @@ define void @trunc_sat_u32u64_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u32u64_maxmin:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vs2r.v v12, (a1)
 ; CHECK-NEXT:    ret
@@ -369,7 +375,9 @@ define void @trunc_sat_u32u64_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u32u64_minmax:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vs2r.v v12, (a1)
 ; CHECK-NEXT:    ret
@@ -438,7 +446,9 @@ define void @trunc_sat_u8u32_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u32_maxmin:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2re32.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -456,7 +466,9 @@ define void @trunc_sat_u8u32_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u32_minmax:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2re32.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -533,7 +545,9 @@ define void @trunc_sat_u8u64_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u64_maxmin:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 0
@@ -553,7 +567,9 @@ define void @trunc_sat_u8u64_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u64_minmax:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll b/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll
index 9c558d23e17e08e..d2f73826e4e9e8f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll
@@ -5,7 +5,9 @@
 define <4 x i8> @test_v4i16_v4i8(<4 x i16> %x) {
 ; CHECK-LABEL: test_v4i16_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <4 x i16> %x, zeroinitializer
@@ -19,7 +21,9 @@ define <4 x i8> @test_v4i16_v4i8(<4 x i16> %x) {
 define <4 x i8> @test_v4i32_v4i8(<4 x i32> %x) {
 ; CHECK-LABEL: test_v4i32_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
@@ -35,7 +39,9 @@ define <4 x i8> @test_v4i32_v4i8(<4 x i32> %x) {
 define <4 x i8> @test_v4i64_v4i8(<4 x i64> %x) {
 ; CHECK-LABEL: test_v4i64_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -53,7 +59,9 @@ define <4 x i8> @test_v4i64_v4i8(<4 x i64> %x) {
 define <4 x i16> @test_v4i32_v4i16(<4 x i32> %x) {
 ; CHECK-LABEL: test_v4i32_v4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <4 x i32> %x, zeroinitializer
@@ -67,7 +75,9 @@ define <4 x i16> @test_v4i32_v4i16(<4 x i32> %x) {
 define <4 x i16> @test_v4i64_v4i16(<4 x i64> %x) {
 ; CHECK-LABEL: test_v4i64_v4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -83,9 +93,10 @@ define <4 x i16> @test_v4i64_v4i16(<4 x i64> %x) {
 define <4 x i32> @test_v4i64_v4i32(<4 x i64> %x) {
 ; CHECK-LABEL: test_v4i64_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vnclipu.wi v10, v8, 0
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v10, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <4 x i64> %x, zeroinitializer
   %b = sext <4 x i1> %a to <4 x i64>
@@ -98,7 +109,9 @@ define <4 x i32> @test_v4i64_v4i32(<4 x i64> %x) {
 define <vscale x 4 x i8> @test_nxv4i16_nxv4i8(<vscale x 4 x i16> %x) {
 ; CHECK-LABEL: test_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <vscale x 4 x i16> %x, zeroinitializer
@@ -112,7 +125,9 @@ define <vscale x 4 x i8> @test_nxv4i16_nxv4i8(<vscale x 4 x i16> %x) {
 define <vscale x 4 x i8> @test_nxv4i32_nxv4i8(<vscale x 4 x i32> %x) {
 ; CHECK-LABEL: test_nxv4i32_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -128,7 +143,9 @@ define <vscale x 4 x i8> @test_nxv4i32_nxv4i8(<vscale x 4 x i32> %x) {
 define <vscale x 4 x i8> @test_nxv4i64_nxv4i8(<vscale x 4 x i64> %x) {
 ; CHECK-LABEL: test_nxv4i64_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 0
@@ -146,9 +163,10 @@ define <vscale x 4 x i8> @test_nxv4i64_nxv4i8(<vscale x 4 x i64> %x) {
 define <vscale x 4 x i16> @test_nxv4i32_nxv4i16(<vscale x 4 x i32> %x) {
 ; CHECK-LABEL: test_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vnclipu.wi v10, v8, 0
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v10, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <vscale x 4 x i32> %x, zeroinitializer
   %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i32>
@@ -161,7 +179,9 @@ define <vscale x 4 x i16> @test_nxv4i32_nxv4i16(<vscale x 4 x i32> %x) {
 define <vscale x 4 x i16> @test_nxv4i64_nxv4i16(<vscale x 4 x i64> %x) {
 ; CHECK-LABEL: test_nxv4i64_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 0
@@ -177,9 +197,10 @@ define <vscale x 4 x i16> @test_nxv4i64_nxv4i16(<vscale x 4 x i64> %x) {
 define <vscale x 4 x i32> @test_nxv4i64_nxv4i32(<vscale x 4 x i64> %x) {
 ; CHECK-LABEL: test_nxv4i64_nxv4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vnclipu.wi v12, v8, 0
-; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v12, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v12, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <vscale x 4 x i64> %x, zeroinitializer
   %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i64>
diff --git a/llvm/test/CodeGen/SPARC/tn0009.mir b/llvm/test/CodeGen/SPARC/tn0009.mir
new file mode 100644
index 000000000000000..f3807940cd30ec4
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0009.mir
@@ -0,0 +1,26 @@
+# RUN: llc %s -mattr=+fix-tn0009 -march=sparc -run-pass=errata-workaround -o - \
+# RUN:   | FileCheck %s
+
+---
+# CHECK:      STrr $o1, $g0, $o0
+# CHECK-NEXT: NOP
+# CHECK-NEXT: NOP
+# CHECK-NEXT: STrr $o1, $g0, $o0
+name:            seqa
+body:             |
+  bb.0.entry:
+    STrr $o1, $g0, $o0
+    NOP
+    STrr $o1, $g0, $o0
+
+---
+# CHECK:      STDrr $i0, $g0, $i0_i1
+# CHECK-NEXT: NOP
+# CHECK-NEXT: STDrr $i0, $g0, $i0_i1
+name:            seqb
+body:             |
+  bb.0.entry:
+    STDrr $i0, $g0, $i0_i1
+    STDrr $i0, $g0, $i0_i1
+
+...
diff --git a/llvm/test/CodeGen/SPARC/tn0010.mir b/llvm/test/CodeGen/SPARC/tn0010.mir
new file mode 100644
index 000000000000000..416dd740cf21cac
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0010.mir
@@ -0,0 +1,79 @@
+# RUN: llc %s -mattr=+fix-tn0010 -march=sparc -run-pass=errata-workaround -o - \
+# RUN:   | FileCheck %s
+
+---
+# CHECK:      LDrr
+# CHECK-NEXT: NOP
+# CHECK-NEXT: CASArr
+# CHECK:      LDrr
+# CHECK-NEXT: NOP
+# CHECK-NEXT: SWAPrr
+# CHECK:      LDrr
+# CHECK-NEXT: NOP
+# CHECK-NEXT: SWAPri
+name:            er-5-1
+registers:
+  - { id: 0, class: intregs }
+body:             |
+  bb.0.entry:
+    %0 = LDrr $o0, $g0
+    %0 = CASArr $o2, $o3, %0, 10
+    %0 = LDrr $g0, $o0
+    %0 = SWAPrr $o2, $o3, %0
+    %0 = LDrr $g0, $o0
+    %0 = SWAPri $o2, 10, %0
+
+---
+# CHECK:      bb.2:
+# CHECK-NEXT: NOP
+# CHECK-NEXT: CASArr
+name:            er-5-2
+registers:
+  - { id: 0, class: intregs }
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    BCOND %bb.2, 10, implicit $icc {
+      %0 = LDrr $g0, $o0
+    }
+
+  bb.1.entry:
+    successors: %bb.2
+    NOP
+
+  bb.2:
+    %0 = CASArr $o1, $o2, %0, 10
+
+---
+# CHECK:      bb.0.entry:
+# CHECK-NEXT: NOP
+# CHECK-NEXT: CASArr
+name:            er-5-3
+registers:
+  - { id: 0, class: intregs }
+body:             |
+  bb.0.entry:
+    %0 = CASArr $o1, $o2, %0, 10
+
+---
+# CHECK:      bb.1.entry:
+# CHECK:      NOP
+# CHECK-NEXT: CASArr
+name:            er-5-4
+registers:
+  - { id: 0, class: intregs }
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    BCOND %bb.2, 10, implicit $icc {
+      %0 = LDrr $g0, $o0
+    }
+
+  bb.1.entry:
+    successors: %bb.2
+    %0 = CASArr $o1, $o2, %0, 10
+
+  bb.2:
+    NOP
+
+...
diff --git a/llvm/test/CodeGen/SPARC/tn0011.ll b/llvm/test/CodeGen/SPARC/tn0011.ll
new file mode 100644
index 000000000000000..e13379eb54f4a98
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0011.ll
@@ -0,0 +1,26 @@
+; RUN: llc < %s -mtriple=sparc -mattr=fix-tn0011,hasleoncasa | FileCheck %s
+
+; CHECK: .p2align 4
+; CHECK-NEXT: casa
+define i32 @test_casarr(i32* %p, i32 %v) {
+entry:
+  %0 = atomicrmw nand i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK: .p2align 4
+; CHECK-NEXT: swap
+define i32 @test_swaprr(i32* %p, i32 %v) {
+entry:
+  %0 = atomicrmw xchg i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK: .p2align 4
+; CHECK-NEXT: swap
+define i32 @test_swapri(i32* %p, i32 %v) {
+entry:
+  %1 = getelementptr inbounds i32, ptr %p, i32 1
+  %2 = atomicrmw xchg ptr %1, i32 %v seq_cst, align 4
+  ret i32 %2
+}
diff --git a/llvm/test/CodeGen/SPARC/tn0012.mir b/llvm/test/CodeGen/SPARC/tn0012.mir
new file mode 100644
index 000000000000000..f6c55b0d9a770d8
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0012.mir
@@ -0,0 +1,65 @@
+# RUN: llc %s -mattr=+fix-tn0012 -march=sparc -run-pass=errata-workaround -o - \
+# RUN:   | FileCheck %s
+
+---
+# CHECK:      bb.2:
+# CHECK:      NOP
+# CHECK-NEXT: $f0 = FADDS $f0, $f0
+name:            er-5-1
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    BCOND %bb.2, 10, implicit $icc {
+      NOP
+    }
+
+  bb.1.entry:
+    successors: %bb.2
+    NOP
+
+  bb.2:
+    $f0 = FADDS $f0, $f0
+
+---
+# CHECK:      bb.2:
+# CHECK:      NOP
+# CHECK-NEXT: FBCOND %bb.3, 22, implicit $fcc0 {
+name:            er-5-2
+body:             |
+  bb.0.entry:
+    successors: %bb.2, %bb.1
+    BCOND %bb.1, 10, implicit $icc {
+      NOP
+    }
+
+  bb.2.entry:
+    successors: %bb.1
+    NOP
+
+  bb.1:
+    successors: %bb.3
+    FBCOND %bb.3, 22, implicit $fcc0 {
+      NOP
+    }
+  bb.3:
+   NOP
+
+---
+# CHECK:      bb.2:
+# CHECK:      NOP
+# CHECK-NEXT: FCMPS $f0, $f0, implicit-def $fcc0
+name:            er-5-3
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    BCOND %bb.2, 10, implicit $icc {
+      NOP
+    }
+
+  bb.1.entry:
+    successors: %bb.2
+    NOP
+
+  bb.2:
+    FCMPS $f0, $f0, implicit-def $fcc0
+...
diff --git a/llvm/test/CodeGen/SPARC/tn0013.mir b/llvm/test/CodeGen/SPARC/tn0013.mir
new file mode 100644
index 000000000000000..068bd0f02b34829
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0013.mir
@@ -0,0 +1,93 @@
+# RUN: llc %s -mattr=+fix-tn0013 -march=sparc -run-pass=errata-workaround -o - \
+# RUN:   | FileCheck %s
+
+---
+# CHECK:      $f0 = FSQRTS $f0
+# CHECK-NEXT: NOP
+# CHECK-NEXT: NOP
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f0 = FDIVS $f4, $f5
+name:            er-8-1
+body:             |
+  bb.0.entry:
+    $f0 = FSQRTS $f0
+    $f3 = FADDS $f1, $f2
+    $f3 = FADDS $f1, $f2
+    $f0 = FDIVS $f4, $f5
+
+---
+# CHECK:      $f0 = FDIVS $f0, $f0
+# CHECK-NEXT: NOP
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f0 = FSQRTS $f4
+name:            er-8-2
+body:             |
+  bb.0.entry:
+    $f0 = FDIVS $f0, $f0
+    $f3 = FADDS $f1, $f2
+    $f3 = FADDS $f1, $f2
+    $f3 = FADDS $f1, $f2
+    $f0 = FSQRTS $f4
+
+---
+# CHECK-NOT:  NOP
+name:            er-9
+body:             |
+  bb.0.entry:
+    $f0 = FSQRTS $f0
+    $f3 = FADDS $f1, $f2
+    $f3 = FADDS $f0, $f2
+    $f3 = FADDS $f1, $f2
+    $f0 = FSQRTS $f0
+
+---
+# CHECK-NOT:  NOP
+name:            er-10
+body:             |
+  bb.0.entry:
+    $f0 = FSQRTS $f0
+    $f4 = FSQRTS $f4
+    $f3 = FADDS $f1, $f2
+    $f0 = FSQRTS $f0
+
+---
+# CHECK:      er-11
+# CHECK:      $f0 = FSQRTS $f0
+# CHECK-NEXT: NOP
+# CHECK:      $f0 = FDIVS $f0, $f0
+# CHECK-NEXT: NOP
+name:            er-11
+body:             |
+  bb.0.entry:
+    successors: %bb.3, %bb.1
+    $f0 = FSQRTS $f0
+    FBCOND %bb.3, 22, implicit $fcc0 {
+      NOP
+    }
+  bb.1:
+    successors: %bb.3, %bb.4
+    $f0 = FDIVS $f0, $f0
+    BCOND %bb.4, 10, implicit $icc {
+      NOP
+    }
+  bb.3:
+    NOP
+  bb.4:
+    NOP
+
+---
+# CHECK:      $f1 = FDIVS $f0, $f1
+# CHECK-NEXT: NOP
+# CHECK-NEXT: STri $i6, -84, $i2
+name:            er-8-3
+body:             |
+  bb.0.entry:
+    $f1 = FDIVS $f0, $f1
+    STri $i6, -84, $i2
+    $f0 = LDFri $i6, -84
+    $f0 = FITOS $f0
+    $f5 = FDIVS $f4, $f0
+...
diff --git a/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll b/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
index 94210d795867a0f..f2ac5268921800c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
@@ -1012,60 +1012,21 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-LABEL: stest_f16i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs.w r7, r0, #-1
-; CHECK-NEXT:    mvn r9, #-2147483648
-; CHECK-NEXT:    sbcs.w r7, r1, r9
-; CHECK-NEXT:    mov.w r10, #-2147483648
-; CHECK-NEXT:    sbcs r7, r2, #0
-; CHECK-NEXT:    sbcs r7, r3, #0
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r3, r3, r7, ne
-; CHECK-NEXT:    csel r2, r2, r7, ne
-; CHECK-NEXT:    mov.w r7, #-1
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r4, r0, r7, ne
-; CHECK-NEXT:    rsbs r0, r4, #0
-; CHECK-NEXT:    sbcs.w r0, r10, r1
-; CHECK-NEXT:    sbcs.w r0, r7, r2
-; CHECK-NEXT:    sbcs.w r0, r7, r3
-; CHECK-NEXT:    cset r5, lt
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r8, r1, r10, ne
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs.w r6, r0, #-1
-; CHECK-NEXT:    sbcs.w r6, r1, r9
-; CHECK-NEXT:    sbcs r6, r2, #0
-; CHECK-NEXT:    sbcs r6, r3, #0
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r3, r3, r6, ne
-; CHECK-NEXT:    csel r2, r2, r6, ne
-; CHECK-NEXT:    rsbs r6, r0, #0
-; CHECK-NEXT:    sbcs.w r6, r10, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
-; CHECK-NEXT:    cset r2, lt
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r3, r4, r5, ne
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r3
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r8
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 9223372036854775807, i128 9223372036854775807>
@@ -1105,46 +1066,28 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64(<2 x half> %x) {
 ; CHECK-LABEL: ustest_f16i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs r5, r2, #1
-; CHECK-NEXT:    mov.w r8, #1
-; CHECK-NEXT:    sbcs r5, r3, #0
-; CHECK-NEXT:    mov.w r7, #0
-; CHECK-NEXT:    cset r5, lt
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r0, r0, r5, ne
-; CHECK-NEXT:    csel r3, r3, r5, ne
-; CHECK-NEXT:    csel r2, r2, r8, ne
-; CHECK-NEXT:    csel r4, r1, r5, ne
+; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    sbcs.w r1, r7, r4
-; CHECK-NEXT:    sbcs.w r1, r7, r2
-; CHECK-NEXT:    sbcs.w r1, r7, r3
+; CHECK-NEXT:    mov.w r5, #0
+; CHECK-NEXT:    sbcs.w r1, r5, r4
+; CHECK-NEXT:    sbcs.w r1, r5, r2
+; CHECK-NEXT:    sbcs.w r1, r5, r3
 ; CHECK-NEXT:    cset r6, lt
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r9, r0, r6, ne
+; CHECK-NEXT:    csel r8, r0, r6, ne
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs r5, r2, #1
-; CHECK-NEXT:    sbcs r5, r3, #0
-; CHECK-NEXT:    cset r5, lt
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r0, r0, r5, ne
-; CHECK-NEXT:    csel r2, r2, r8, ne
-; CHECK-NEXT:    csel r3, r3, r5, ne
-; CHECK-NEXT:    csel r1, r1, r5, ne
-; CHECK-NEXT:    rsbs r5, r0, #0
-; CHECK-NEXT:    sbcs.w r5, r7, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
+; CHECK-NEXT:    rsbs r7, r0, #0
+; CHECK-NEXT:    sbcs.w r7, r5, r1
+; CHECK-NEXT:    sbcs.w r2, r5, r2
+; CHECK-NEXT:    sbcs.w r2, r5, r3
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
@@ -1152,11 +1095,10 @@ define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64(<2 x half> %x) {
 ; CHECK-NEXT:    csel r3, r4, r6, ne
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r9
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r8
 ; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -2119,60 +2061,21 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-LABEL: stest_f16i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs.w r7, r0, #-1
-; CHECK-NEXT:    mvn r9, #-2147483648
-; CHECK-NEXT:    sbcs.w r7, r1, r9
-; CHECK-NEXT:    mov.w r10, #-2147483648
-; CHECK-NEXT:    sbcs r7, r2, #0
-; CHECK-NEXT:    sbcs r7, r3, #0
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r3, r3, r7, ne
-; CHECK-NEXT:    csel r2, r2, r7, ne
-; CHECK-NEXT:    mov.w r7, #-1
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r4, r0, r7, ne
-; CHECK-NEXT:    rsbs r0, r4, #0
-; CHECK-NEXT:    sbcs.w r0, r10, r1
-; CHECK-NEXT:    sbcs.w r0, r7, r2
-; CHECK-NEXT:    sbcs.w r0, r7, r3
-; CHECK-NEXT:    cset r5, lt
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r8, r1, r10, ne
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs.w r6, r0, #-1
-; CHECK-NEXT:    sbcs.w r6, r1, r9
-; CHECK-NEXT:    sbcs r6, r2, #0
-; CHECK-NEXT:    sbcs r6, r3, #0
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r3, r3, r6, ne
-; CHECK-NEXT:    csel r2, r2, r6, ne
-; CHECK-NEXT:    rsbs r6, r0, #0
-; CHECK-NEXT:    sbcs.w r6, r10, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
-; CHECK-NEXT:    cset r2, lt
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r3, r4, r5, ne
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r3
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r8
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 9223372036854775807, i128 9223372036854775807>)
@@ -2209,51 +2112,34 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-LABEL: ustest_f16i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    subs r1, r2, #1
-; CHECK-NEXT:    sbcs r1, r3, #0
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r5, r0, r6, ne
-; CHECK-NEXT:    csel r7, r3, r6, ne
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r5, #0
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r3
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs r2, #1
-; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    cset r2, lt
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r3, r3, r2, ne
-; CHECK-NEXT:    csel r0, r0, r2, ne
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it mi
+; CHECK-NEXT:    movmi r4, #0
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    it mi
 ; CHECK-NEXT:    movmi r0, #0
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    csel r6, r4, r6, ne
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
 ; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r6, #0
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r2, ne
+; CHECK-NEXT:    movmi r5, #0
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    it mi
 ; CHECK-NEXT:    movmi r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r6
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
diff --git a/llvm/test/CodeGen/X86/abds-neg.ll b/llvm/test/CodeGen/X86/abds-neg.ll
index 541ed1cbf4fe89b..b72bb46209d2d66 100644
--- a/llvm/test/CodeGen/X86/abds-neg.ll
+++ b/llvm/test/CodeGen/X86/abds-neg.ll
@@ -139,28 +139,25 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i16_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i16_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-NEXT:    movswq %di, %rcx
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movswl %di, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %ecx, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %aext = sext i16 %a to i64
   %bext = sext i32 %b to i64
@@ -205,26 +202,22 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i32 %b to i64
@@ -238,27 +231,23 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-NEXT:    movswq %si, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movswl %si, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %eax, %ecx
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    cmovll %ecx, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i16 %b to i64
@@ -272,26 +261,22 @@ define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_undef:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i32 %b to i64
@@ -332,9 +317,8 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    negq %rax
 ; X64-NEXT:    retq
   %aext = sext i64 %a to i128
@@ -376,9 +360,8 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    negq %rax
 ; X64-NEXT:    retq
   %aext = sext i64 %a to i128
diff --git a/llvm/test/CodeGen/X86/abds.ll b/llvm/test/CodeGen/X86/abds.ll
index fd47de5af05ab4d..e972ef3787e4d59 100644
--- a/llvm/test/CodeGen/X86/abds.ll
+++ b/llvm/test/CodeGen/X86/abds.ll
@@ -129,26 +129,23 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i16_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i16_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-NEXT:    movswq %di, %rcx
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    movswl %di, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %ecx, %esi
+; X64-NEXT:    cmovgel %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %aext = sext i16 %a to i64
   %bext = sext i32 %b to i64
@@ -191,24 +188,20 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i32 %b to i64
@@ -221,25 +214,21 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-NEXT:    movswq %si, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movswl %si, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %eax, %ecx
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    cmovll %ecx, %eax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i16 %b to i64
@@ -252,24 +241,20 @@ define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_undef:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i32 %b to i64
@@ -306,9 +291,8 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %aext = sext i64 %a to i128
   %bext = sext i64 %b to i128
@@ -345,9 +329,8 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %aext = sext i64 %a to i128
   %bext = sext i64 %b to i128
@@ -542,24 +525,20 @@ define i16 @abd_minmax_i16(i16 %a, i16 %b) nounwind {
 define i32 @abd_minmax_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_minmax_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_minmax_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    retq
   %min = call i32 @llvm.smin.i32(i32 %a, i32 %b)
   %max = call i32 @llvm.smax.i32(i32 %a, i32 %b)
@@ -594,9 +573,8 @@ define i64 @abd_minmax_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %min = call i64 @llvm.smin.i64(i64 %a, i64 %b)
   %max = call i64 @llvm.smax.i64(i64 %a, i64 %b)
diff --git a/llvm/test/CodeGen/X86/abdu-neg.ll b/llvm/test/CodeGen/X86/abdu-neg.ll
index fe4525009ad5c81..9cb3e5e8bf0c263 100644
--- a/llvm/test/CodeGen/X86/abdu-neg.ll
+++ b/llvm/test/CodeGen/X86/abdu-neg.ll
@@ -139,27 +139,25 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i16_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i16_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    movzwl %di, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %ecx, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %aext = zext i16 %a to i64
   %bext = zext i32 %b to i64
@@ -204,26 +202,22 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i32 %b to i64
@@ -237,26 +231,23 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %ecx
 ; X64-NEXT:    movzwl %si, %eax
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %eax, %ecx
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    cmovbl %ecx, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i16 %b to i64
@@ -270,26 +261,22 @@ define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_undef:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i32 %b to i64
@@ -324,9 +311,8 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    negq %rax
 ; X64-NEXT:    retq
   %aext = zext i64 %a to i128
@@ -362,9 +348,8 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    negq %rax
 ; X64-NEXT:    retq
   %aext = zext i64 %a to i128
diff --git a/llvm/test/CodeGen/X86/abdu.ll b/llvm/test/CodeGen/X86/abdu.ll
index 2e2a1debc38cb13..e808e0f21babf28 100644
--- a/llvm/test/CodeGen/X86/abdu.ll
+++ b/llvm/test/CodeGen/X86/abdu.ll
@@ -129,25 +129,23 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i16_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i16_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    movzwl %di, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %ecx, %esi
+; X64-NEXT:    cmovael %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %aext = zext i16 %a to i64
   %bext = zext i32 %b to i64
@@ -190,24 +188,20 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i32 %b to i64
@@ -220,24 +214,21 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %ecx
 ; X64-NEXT:    movzwl %si, %eax
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %eax, %ecx
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    cmovbl %ecx, %eax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i16 %b to i64
@@ -250,24 +241,20 @@ define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_undef:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i32 %b to i64
@@ -296,9 +283,8 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    retq
   %aext = zext i64 %a to i128
   %bext = zext i64 %b to i128
@@ -327,9 +313,8 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    retq
   %aext = zext i64 %a to i128
   %bext = zext i64 %b to i128
@@ -512,24 +497,20 @@ define i16 @abd_minmax_i16(i16 %a, i16 %b) nounwind {
 define i32 @abd_minmax_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_minmax_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_minmax_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    retq
   %min = call i32 @llvm.umin.i32(i32 %a, i32 %b)
   %max = call i32 @llvm.umax.i32(i32 %a, i32 %b)
@@ -556,9 +537,8 @@ define i64 @abd_minmax_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    retq
   %min = call i64 @llvm.umin.i64(i64 %a, i64 %b)
   %max = call i64 @llvm.umax.i64(i64 %a, i64 %b)
diff --git a/llvm/test/CodeGen/X86/avx512f-large-stack.ll b/llvm/test/CodeGen/X86/avx512f-large-stack.ll
new file mode 100644
index 000000000000000..3cb5391c56abf5c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512f-large-stack.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp --version 4
+; RUN: llc -O0 -mtriple=x86_64 -mattr=+avx512f -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
+define void @f(i16 %LGV2, i1 %LGV3) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %BB
+; CHECK-NEXT:    subq $2147483528, %rsp # imm = 0x7FFFFF88
+; CHECK-NEXT:    .cfi_def_cfa_offset 2147483536
+; CHECK-NEXT:    movb %sil, %cl
+; CHECK-NEXT:    movw %di, %ax
+; CHECK-NEXT:    movswq %ax, %rax
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    movabsq $-2147483768, %rdx # imm = 0xFFFFFFFF7FFFFF88
+; CHECK-NEXT:    movb %cl, (%rsp,%rdx)
+; CHECK-NEXT:    addq $2147483528, %rsp # imm = 0x7FFFFF88
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+BB:
+  %A = alloca i1, i33 2147483648, align 1
+  %G = getelementptr i1, ptr %A, i16 %LGV2
+  %G4 = getelementptr i1, ptr %G, i32 -2147483648
+  store i1 %LGV3, ptr %G4, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index ec76e8b05678b0b..3759909a2ccc8e3 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -759,347 +759,21 @@ define <32 x bfloat> @pr63017_2() nounwind {
 ;
 ; SSE2-LABEL: pr63017_2:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pushq %r14
-; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    subq $200, %rsp
 ; SSE2-NEXT:    xorl %eax, %eax
 ; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    jne .LBB12_1
 ; SSE2-NEXT:  # %bb.2: # %cond.load
 ; SSE2-NEXT:    movzwl (%rax), %eax
 ; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movdqa %xmm0, %xmm15
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movdqa %xmm0, %xmm13
-; SSE2-NEXT:    movdqa %xmm0, %xmm14
-; SSE2-NEXT:    movdqa %xmm0, %xmm11
-; SSE2-NEXT:    movdqa %xmm0, %xmm12
-; SSE2-NEXT:    movdqa %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm0, %xmm10
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    jmp .LBB12_3
 ; SSE2-NEXT:  .LBB12_1:
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movdqa %xmm2, %xmm15
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movdqa %xmm2, %xmm13
-; SSE2-NEXT:    movdqa %xmm2, %xmm14
-; SSE2-NEXT:    movdqa %xmm2, %xmm11
-; SSE2-NEXT:    movdqa %xmm2, %xmm12
-; SSE2-NEXT:    movdqa %xmm2, %xmm9
-; SSE2-NEXT:    movdqa %xmm2, %xmm10
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:  .LBB12_3: # %else
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_5
-; SSE2-NEXT:  # %bb.4: # %cond.load1
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:  .LBB12_5: # %else2
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_7
-; SSE2-NEXT:  # %bb.6: # %cond.load4
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_7: # %else5
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_9
-; SSE2-NEXT:  # %bb.8: # %cond.load7
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_9: # %else8
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_11
-; SSE2-NEXT:  # %bb.10: # %cond.load10
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_11: # %else11
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_13
-; SSE2-NEXT:  # %bb.12: # %cond.load13
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_13: # %else14
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_15
-; SSE2-NEXT:  # %bb.14: # %cond.load16
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_15: # %else17
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_17
-; SSE2-NEXT:  # %bb.16: # %cond.load19
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_17: # %else20
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_19
-; SSE2-NEXT:  # %bb.18: # %cond.load22
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_19: # %else23
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_21
-; SSE2-NEXT:  # %bb.20: # %cond.load25
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_21: # %else26
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_23
-; SSE2-NEXT:  # %bb.22: # %cond.load28
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_23: # %else29
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_25
-; SSE2-NEXT:  # %bb.24: # %cond.load31
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_25: # %else32
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_27
-; SSE2-NEXT:  # %bb.26: # %cond.load34
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_27: # %else35
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_29
-; SSE2-NEXT:  # %bb.28: # %cond.load37
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_29: # %else38
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_31
-; SSE2-NEXT:  # %bb.30: # %cond.load40
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_31: # %else41
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_33
-; SSE2-NEXT:  # %bb.32: # %cond.load43
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_33: # %else44
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_35
-; SSE2-NEXT:  # %bb.34: # %cond.load46
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm15
-; SSE2-NEXT:  .LBB12_35: # %else47
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_37
-; SSE2-NEXT:  # %bb.36: # %cond.load49
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_37: # %else50
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_39
-; SSE2-NEXT:  # %bb.38: # %cond.load52
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm13
-; SSE2-NEXT:  .LBB12_39: # %else53
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_41
-; SSE2-NEXT:  # %bb.40: # %cond.load55
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm14
-; SSE2-NEXT:  .LBB12_41: # %else56
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_43
-; SSE2-NEXT:  # %bb.42: # %cond.load58
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm11
-; SSE2-NEXT:  .LBB12_43: # %else59
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_45
-; SSE2-NEXT:  # %bb.44: # %cond.load61
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm12
-; SSE2-NEXT:  .LBB12_45: # %else62
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_47
-; SSE2-NEXT:  # %bb.46: # %cond.load64
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm9
-; SSE2-NEXT:  .LBB12_47: # %else65
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_49
-; SSE2-NEXT:  # %bb.48: # %cond.load67
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm10
-; SSE2-NEXT:  .LBB12_49: # %else68
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_51
-; SSE2-NEXT:  # %bb.50: # %cond.load70
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm7
-; SSE2-NEXT:  .LBB12_51: # %else71
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_53
-; SSE2-NEXT:  # %bb.52: # %cond.load73
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm8
-; SSE2-NEXT:  .LBB12_53: # %else74
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_55
-; SSE2-NEXT:  # %bb.54: # %cond.load76
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:  .LBB12_55: # %else77
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_57
-; SSE2-NEXT:  # %bb.56: # %cond.load79
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm6
-; SSE2-NEXT:  .LBB12_57: # %else80
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_59
-; SSE2-NEXT:  # %bb.58: # %cond.load82
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:  .LBB12_59: # %else83
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_61
-; SSE2-NEXT:  # %bb.60: # %cond.load85
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:  .LBB12_61: # %else86
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_63
-; SSE2-NEXT:  # %bb.62: # %cond.load88
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:  .LBB12_63: # %else89
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    movd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    jne .LBB12_64
-; SSE2-NEXT:  # %bb.65: # %cond.load91
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    jmp .LBB12_66
-; SSE2-NEXT:  .LBB12_64:
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:  .LBB12_66: # %else92
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; SSE2-NEXT:  .LBB12_3:
+; SSE2-NEXT:    pushq %r14
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    subq $88, %rsp
+; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
 ; SSE2-NEXT:    callq __truncsfbf2@PLT
 ; SSE2-NEXT:    pextrw $0, %xmm0, %ebx
 ; SSE2-NEXT:    shll $16, %ebx
@@ -1316,7 +990,7 @@ define <32 x bfloat> @pr63017_2() nounwind {
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE2-NEXT:    addq $200, %rsp
+; SSE2-NEXT:    addq $88, %rsp
 ; SSE2-NEXT:    popq %rbx
 ; SSE2-NEXT:    popq %r14
 ; SSE2-NEXT:    retq
@@ -1329,250 +1003,14 @@ define <32 x bfloat> @pr63017_2() nounwind {
 ;
 ; AVXNC-LABEL: pr63017_2:
 ; AVXNC:       # %bb.0:
-; AVXNC-NEXT:    vpbroadcastw {{.*#+}} ymm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
+; AVXNC-NEXT:    vbroadcastss {{.*#+}} ymm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
 ; AVXNC-NEXT:    xorl %eax, %eax
 ; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    vmovdqa %ymm0, %ymm1
 ; AVXNC-NEXT:    jne .LBB12_2
 ; AVXNC-NEXT:  # %bb.1: # %cond.load
-; AVXNC-NEXT:    vpbroadcastw {{.*#+}} ymm1 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
-; AVXNC-NEXT:    vpbroadcastw {{.*#+}} ymm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
-; AVXNC-NEXT:    vpinsrw $0, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_2: # %else
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_4
-; AVXNC-NEXT:  # %bb.3: # %cond.load1
-; AVXNC-NEXT:    vpinsrw $1, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_4: # %else2
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_6
-; AVXNC-NEXT:  # %bb.5: # %cond.load4
-; AVXNC-NEXT:    vpinsrw $2, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_6: # %else5
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_8
-; AVXNC-NEXT:  # %bb.7: # %cond.load7
-; AVXNC-NEXT:    vpinsrw $3, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_8: # %else8
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_10
-; AVXNC-NEXT:  # %bb.9: # %cond.load10
-; AVXNC-NEXT:    vpinsrw $4, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_10: # %else11
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_12
-; AVXNC-NEXT:  # %bb.11: # %cond.load13
-; AVXNC-NEXT:    vpinsrw $5, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_12: # %else14
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_14
-; AVXNC-NEXT:  # %bb.13: # %cond.load16
-; AVXNC-NEXT:    vpinsrw $6, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_14: # %else17
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_16
-; AVXNC-NEXT:  # %bb.15: # %cond.load19
-; AVXNC-NEXT:    vpinsrw $7, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_16: # %else20
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_18
-; AVXNC-NEXT:  # %bb.17: # %cond.load22
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_18: # %else23
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_20
-; AVXNC-NEXT:  # %bb.19: # %cond.load25
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2,3,4,5,6,7,8],ymm2[9],ymm0[10,11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_20: # %else26
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_22
-; AVXNC-NEXT:  # %bb.21: # %cond.load28
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1],ymm2[2],ymm0[3,4,5,6,7,8,9],ymm2[10],ymm0[11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_22: # %else29
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_24
-; AVXNC-NEXT:  # %bb.23: # %cond.load31
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6,7,8,9,10],ymm2[11],ymm0[12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_24: # %else32
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_26
-; AVXNC-NEXT:  # %bb.25: # %cond.load34
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm2[4],ymm0[5,6,7,8,9,10,11],ymm2[12],ymm0[13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_26: # %else35
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_28
-; AVXNC-NEXT:  # %bb.27: # %cond.load37
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7,8,9,10,11,12],ymm2[13],ymm0[14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_28: # %else38
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_30
-; AVXNC-NEXT:  # %bb.29: # %cond.load40
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7,8,9,10,11,12,13],ymm2[14],ymm0[15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_30: # %else41
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_32
-; AVXNC-NEXT:  # %bb.31: # %cond.load43
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5,6],ymm2[7],ymm0[8,9,10,11,12,13,14],ymm2[15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_32: # %else44
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_34
-; AVXNC-NEXT:  # %bb.33: # %cond.load46
-; AVXNC-NEXT:    vpinsrw $0, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_34: # %else47
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_36
-; AVXNC-NEXT:  # %bb.35: # %cond.load49
-; AVXNC-NEXT:    vpinsrw $1, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_36: # %else50
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_38
-; AVXNC-NEXT:  # %bb.37: # %cond.load52
-; AVXNC-NEXT:    vpinsrw $2, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_38: # %else53
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_40
-; AVXNC-NEXT:  # %bb.39: # %cond.load55
-; AVXNC-NEXT:    vpinsrw $3, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_40: # %else56
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_42
-; AVXNC-NEXT:  # %bb.41: # %cond.load58
-; AVXNC-NEXT:    vpinsrw $4, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_42: # %else59
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_44
-; AVXNC-NEXT:  # %bb.43: # %cond.load61
-; AVXNC-NEXT:    vpinsrw $5, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_44: # %else62
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_46
-; AVXNC-NEXT:  # %bb.45: # %cond.load64
-; AVXNC-NEXT:    vpinsrw $6, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_46: # %else65
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_48
-; AVXNC-NEXT:  # %bb.47: # %cond.load67
-; AVXNC-NEXT:    vpinsrw $7, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_48: # %else68
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_50
-; AVXNC-NEXT:  # %bb.49: # %cond.load70
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_50: # %else71
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_52
-; AVXNC-NEXT:  # %bb.51: # %cond.load73
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3,4,5,6,7,8],ymm2[9],ymm1[10,11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_52: # %else74
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_54
-; AVXNC-NEXT:  # %bb.53: # %cond.load76
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6,7,8,9],ymm2[10],ymm1[11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_54: # %else77
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_56
-; AVXNC-NEXT:  # %bb.55: # %cond.load79
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7,8,9,10],ymm2[11],ymm1[12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_56: # %else80
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_58
-; AVXNC-NEXT:  # %bb.57: # %cond.load82
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm2[4],ymm1[5,6,7,8,9,10,11],ymm2[12],ymm1[13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_58: # %else83
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_60
-; AVXNC-NEXT:  # %bb.59: # %cond.load85
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6,7,8,9,10,11,12],ymm2[13],ymm1[14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_60: # %else86
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_62
-; AVXNC-NEXT:  # %bb.61: # %cond.load88
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5],ymm2[6],ymm1[7,8,9,10,11,12,13],ymm2[14],ymm1[15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_62: # %else89
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_64
-; AVXNC-NEXT:  # %bb.63: # %cond.load91
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5,6],ymm2[7],ymm1[8,9,10,11,12,13,14],ymm2[15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_64: # %else92
+; AVXNC-NEXT:    vmovups (%rax), %ymm0
+; AVXNC-NEXT:  .LBB12_2:
+; AVXNC-NEXT:    vmovaps %ymm0, %ymm1
 ; AVXNC-NEXT:    retq
   %1 = call <32 x bfloat> @llvm.masked.load.v32bf16.p0(ptr poison, i32 2, <32 x i1> poison, <32 x bfloat> <bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80>)
   ret <32 x bfloat> %1
diff --git a/llvm/test/CodeGen/X86/clobber_frame_ptr.ll b/llvm/test/CodeGen/X86/clobber_frame_ptr.ll
index 6209e1a85e9e197..fd8ba7feb9f48fb 100644
--- a/llvm/test/CodeGen/X86/clobber_frame_ptr.ll
+++ b/llvm/test/CodeGen/X86/clobber_frame_ptr.ll
@@ -157,3 +157,46 @@ define i64 @test3(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6,
   %x = call cc 11 i64 @hipe2(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7)
   ret i64 %x
 }
+
+@buf = dso_local global [20 x ptr] zeroinitializer, align 16
+
+; longjmp modifies fp, it is expected behavior, wo should not save/restore fp
+; around it.
+define void @test4() {
+; CHECK-LABEL: test4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    andq $-16, %rsp
+; CHECK-NEXT:    subq $16, %rsp
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_remember_state
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x06, 0x77, 0x08, 0x06, 0x11, 0x10, 0x22 #
+; CHECK-NEXT:    xorl %r13d, %r13d
+; CHECK-NEXT:    callq external@PLT
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_restore_state
+; CHECK-NEXT:    movq buf(%rip), %rbp
+; CHECK-NEXT:    movq buf+8(%rip), %rax
+; CHECK-NEXT:    movq buf+16(%rip), %rsp
+; CHECK-NEXT:    jmpq *%rax
+entry:
+  %x = call ghccc i32 @external(i32 0)
+  call void @llvm.eh.sjlj.longjmp(ptr @buf)
+  unreachable
+}
diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
index 55502f37da1a627..41d9a867c0a9609 100644
--- a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
@@ -11,6 +11,8 @@ declare float @llvm.minimum.f32(float, float)
 declare double @llvm.minimum.f64(double, double)
 declare <2 x double> @llvm.minimum.v2f64(<2 x double>, <2 x double>)
 declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
+declare <4 x half> @llvm.maximum.v4f16(<4 x half>, <4 x half>)
+declare <4 x bfloat> @llvm.maximum.v4bf16(<4 x bfloat>, <4 x bfloat>)
 
 ;
 ; fmaximum
@@ -1381,3 +1383,1101 @@ define <4 x float> @test_fmaximum_v4f32_splat(<4 x float> %x, float %y) {
   %r = call <4 x float> @llvm.maximum.v4f32(<4 x float> %x, <4 x float> %vec) readnone
   ret <4 x float> %r
 }
+
+define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) {
+; SSE2-LABEL: test_fmaximum_v4f16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    subq $104, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 112
+; SSE2-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    js .LBB33_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:  .LBB33_2:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:    andps %xmm2, %xmm3
+; SSE2-NEXT:    js .LBB33_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:  .LBB33_4:
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    maxss %xmm4, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm3, %xmm0
+; SSE2-NEXT:    callq __truncsfhf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    js .LBB33_6
+; SSE2-NEXT:  # %bb.5:
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:  .LBB33_6:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:    andps %xmm2, %xmm3
+; SSE2-NEXT:    js .LBB33_8
+; SSE2-NEXT:  # %bb.7:
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:  .LBB33_8:
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    psrlq $48, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    psrlq $48, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    maxss %xmm4, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm3, %xmm0
+; SSE2-NEXT:    callq __truncsfhf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    js .LBB33_10
+; SSE2-NEXT:  # %bb.9:
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:  .LBB33_10:
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm1
+; SSE2-NEXT:    movaps %xmm1, %xmm3
+; SSE2-NEXT:    andps %xmm2, %xmm3
+; SSE2-NEXT:    js .LBB33_12
+; SSE2-NEXT:  # %bb.11:
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:  .LBB33_12:
+; SSE2-NEXT:    maxss %xmm4, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm1
+; SSE2-NEXT:    orps %xmm3, %xmm1
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    callq __truncsfhf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movss %xmm0, (%rsp) # 4-byte Spill
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd (%rsp), %xmm4 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    js .LBB33_14
+; SSE2-NEXT:  # %bb.13:
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:  .LBB33_14:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:    andps %xmm2, %xmm3
+; SSE2-NEXT:    js .LBB33_16
+; SSE2-NEXT:  # %bb.15:
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:  .LBB33_16:
+; SSE2-NEXT:    maxss %xmm4, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm3, %xmm0
+; SSE2-NEXT:    callq __truncsfhf2@PLT
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; SSE2-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    addq $104, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 8
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_v4f16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    subq $120, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 128
+; AVX1-NEXT:    vmovaps %xmm0, %xmm2
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpsrld $16, %xmm2, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm1, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vpsrld $16, %xmm1, %xmm0
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB33_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-NEXT:    jmp .LBB33_3
+; AVX1-NEXT:  .LBB33_1:
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:  .LBB33_3:
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm2, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    callq __truncsfhf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB33_4
+; AVX1-NEXT:  # %bb.5:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm2 # 16-byte Reload
+; AVX1-NEXT:    jmp .LBB33_6
+; AVX1-NEXT:  .LBB33_4:
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:  .LBB33_6:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm2, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    callq __truncsfhf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB33_7
+; AVX1-NEXT:  # %bb.8:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm2 # 16-byte Reload
+; AVX1-NEXT:    jmp .LBB33_9
+; AVX1-NEXT:  .LBB33_7:
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:  .LBB33_9:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm2, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    callq __truncsfhf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB33_10
+; AVX1-NEXT:  # %bb.11:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-NEXT:    jmp .LBB33_12
+; AVX1-NEXT:  .LBB33_10:
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:  .LBB33_12:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm2, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    callq __truncsfhf2@PLT
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; AVX1-NEXT:    addq $120, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fmaximum_v4f16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    pushq %rbp
+; AVX512-NEXT:    .cfi_def_cfa_offset 16
+; AVX512-NEXT:    pushq %r15
+; AVX512-NEXT:    .cfi_def_cfa_offset 24
+; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    .cfi_def_cfa_offset 32
+; AVX512-NEXT:    pushq %r13
+; AVX512-NEXT:    .cfi_def_cfa_offset 40
+; AVX512-NEXT:    pushq %r12
+; AVX512-NEXT:    .cfi_def_cfa_offset 48
+; AVX512-NEXT:    pushq %rbx
+; AVX512-NEXT:    .cfi_def_cfa_offset 56
+; AVX512-NEXT:    .cfi_offset %rbx, -56
+; AVX512-NEXT:    .cfi_offset %r12, -48
+; AVX512-NEXT:    .cfi_offset %r13, -40
+; AVX512-NEXT:    .cfi_offset %r14, -32
+; AVX512-NEXT:    .cfi_offset %r15, -24
+; AVX512-NEXT:    .cfi_offset %rbp, -16
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $65535, %ecx # imm = 0xFFFF
+; AVX512-NEXT:    movl $0, %edx
+; AVX512-NEXT:    cmovpl %ecx, %edx
+; AVX512-NEXT:    movl $0, %edi
+; AVX512-NEXT:    cmoval %ecx, %edi
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %esi
+; AVX512-NEXT:    cmovpl %ecx, %esi
+; AVX512-NEXT:    movl $0, %r9d
+; AVX512-NEXT:    cmoval %ecx, %r9d
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm2 = xmm1[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r8d
+; AVX512-NEXT:    cmovpl %ecx, %r8d
+; AVX512-NEXT:    movl $0, %r11d
+; AVX512-NEXT:    cmoval %ecx, %r11d
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm1[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm0[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r10d
+; AVX512-NEXT:    cmovpl %ecx, %r10d
+; AVX512-NEXT:    movl $0, %ebp
+; AVX512-NEXT:    cmoval %ecx, %ebp
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %ebx
+; AVX512-NEXT:    cmovpl %ecx, %ebx
+; AVX512-NEXT:    movl $0, %r14d
+; AVX512-NEXT:    cmoval %ecx, %r14d
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm1[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm0[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r15d
+; AVX512-NEXT:    cmovpl %ecx, %r15d
+; AVX512-NEXT:    movl $0, %r12d
+; AVX512-NEXT:    cmoval %ecx, %r12d
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm2
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r13d
+; AVX512-NEXT:    cmoval %ecx, %r13d
+; AVX512-NEXT:    vmovd %r13d, %xmm2
+; AVX512-NEXT:    vpinsrw $1, %r12d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $2, %r14d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $3, %ebp, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $4, %r11d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $5, %r9d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $6, %edi, %xmm2, %xmm2
+; AVX512-NEXT:    movl $0, %edi
+; AVX512-NEXT:    cmovpl %ecx, %edi
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
+; AVX512-NEXT:    vucomiss %xmm3, %xmm4
+; AVX512-NEXT:    movl $0, %r9d
+; AVX512-NEXT:    cmoval %ecx, %r9d
+; AVX512-NEXT:    vpinsrw $7, %r9d, %xmm2, %xmm2
+; AVX512-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm2
+; AVX512-NEXT:    vmovd %edi, %xmm3
+; AVX512-NEXT:    vpinsrw $1, %r15d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $2, %ebx, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $3, %r10d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $4, %r8d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $5, %esi, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $6, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    movl $0, %edx
+; AVX512-NEXT:    cmovpl %ecx, %edx
+; AVX512-NEXT:    vpinsrw $7, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
+; AVX512-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm2[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT:    vucomiss %xmm4, %xmm3
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm3
+; AVX512-NEXT:    vucomiss %xmm4, %xmm3
+; AVX512-NEXT:    movl $65535, %esi # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %esi
+; AVX512-NEXT:    cmovpl %eax, %esi
+; AVX512-NEXT:    vmovd %esi, %xmm3
+; AVX512-NEXT:    vpinsrw $1, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $2, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm2[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $3, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $4, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $5, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $6, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    cmovnel %eax, %ecx
+; AVX512-NEXT:    cmovpl %eax, %ecx
+; AVX512-NEXT:    vpinsrw $7, %ecx, %xmm3, %xmm3
+; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm5
+; AVX512-NEXT:    vpblendvb %xmm5, %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    vpcmpeqw %xmm4, %xmm1, %xmm4
+; AVX512-NEXT:    vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpblendvb %xmm3, %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    .cfi_def_cfa_offset 48
+; AVX512-NEXT:    popq %r12
+; AVX512-NEXT:    .cfi_def_cfa_offset 40
+; AVX512-NEXT:    popq %r13
+; AVX512-NEXT:    .cfi_def_cfa_offset 32
+; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    .cfi_def_cfa_offset 24
+; AVX512-NEXT:    popq %r15
+; AVX512-NEXT:    .cfi_def_cfa_offset 16
+; AVX512-NEXT:    popq %rbp
+; AVX512-NEXT:    .cfi_def_cfa_offset 8
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_v4f16:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $164, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 168
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpsrlq $48, %xmm1, %xmm0
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpsrld $16, %xmm2, %xmm0
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpsrld $16, %xmm1, %xmm0
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpextrw $0, %xmm1, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm2, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB33_3
+; X86-NEXT:  .LBB33_1:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB33_3:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm1, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_4
+; X86-NEXT:  # %bb.5:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB33_6
+; X86-NEXT:  .LBB33_4:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:  .LBB33_6:
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfhf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    calll __truncsfhf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm1, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_7
+; X86-NEXT:  # %bb.8:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB33_9
+; X86-NEXT:  .LBB33_7:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:  .LBB33_9:
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm1, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_10
+; X86-NEXT:  # %bb.11:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB33_12
+; X86-NEXT:  .LBB33_10:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:  .LBB33_12:
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfhf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovd %xmm0, (%esp)
+; X86-NEXT:    calll __truncsfhf2
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; X86-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; X86-NEXT:    addl $164, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+  %r = call <4 x half> @llvm.maximum.v4f16(<4 x half> %x, <4 x half> %y)
+  ret <4 x half> %r
+}
+
+define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
+; SSE2-LABEL: test_fmaximum_v4bf16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pushq %rbp
+; SSE2-NEXT:    .cfi_def_cfa_offset 16
+; SSE2-NEXT:    pushq %r15
+; SSE2-NEXT:    .cfi_def_cfa_offset 24
+; SSE2-NEXT:    pushq %r14
+; SSE2-NEXT:    .cfi_def_cfa_offset 32
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    .cfi_def_cfa_offset 40
+; SSE2-NEXT:    subq $56, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 96
+; SSE2-NEXT:    .cfi_offset %rbx, -40
+; SSE2-NEXT:    .cfi_offset %r14, -32
+; SSE2-NEXT:    .cfi_offset %r15, -24
+; SSE2-NEXT:    .cfi_offset %rbp, -16
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pextrw $0, %xmm1, %r14d
+; SSE2-NEXT:    pextrw $0, %xmm0, %r15d
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    pextrw $0, %xmm0, %eax
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    pextrw $0, %xmm0, %ecx
+; SSE2-NEXT:    shll $16, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    testl %ecx, %ecx
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    js .LBB34_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:  .LBB34_2:
+; SSE2-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; SSE2-NEXT:    movdqa %xmm5, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1,1,1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm6
+; SSE2-NEXT:    andps %xmm1, %xmm6
+; SSE2-NEXT:    js .LBB34_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:  .LBB34_4:
+; SSE2-NEXT:    pextrw $0, %xmm4, %ebp
+; SSE2-NEXT:    pextrw $0, %xmm5, %ebx
+; SSE2-NEXT:    maxss %xmm2, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm0
+; SSE2-NEXT:    orps %xmm6, %xmm0
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    shll $16, %r15d
+; SSE2-NEXT:    movd %r15d, %xmm3
+; SSE2-NEXT:    shll $16, %r14d
+; SSE2-NEXT:    movd %r14d, %xmm2
+; SSE2-NEXT:    testl %r15d, %r15d
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    js .LBB34_6
+; SSE2-NEXT:  # %bb.5:
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:  .LBB34_6:
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; SSE2-NEXT:    psrlq $48, %xmm5
+; SSE2-NEXT:    movdqa (%rsp), %xmm6 # 16-byte Reload
+; SSE2-NEXT:    psrlq $48, %xmm6
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm1, %xmm4
+; SSE2-NEXT:    js .LBB34_8
+; SSE2-NEXT:  # %bb.7:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:  .LBB34_8:
+; SSE2-NEXT:    pextrw $0, %xmm5, %r15d
+; SSE2-NEXT:    pextrw $0, %xmm6, %r14d
+; SSE2-NEXT:    maxss %xmm2, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    shll $16, %ebx
+; SSE2-NEXT:    movd %ebx, %xmm1
+; SSE2-NEXT:    shll $16, %ebp
+; SSE2-NEXT:    movd %ebp, %xmm3
+; SSE2-NEXT:    testl %ebx, %ebx
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    js .LBB34_10
+; SSE2-NEXT:  # %bb.9:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:  .LBB34_10:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm2, %xmm4
+; SSE2-NEXT:    js .LBB34_12
+; SSE2-NEXT:  # %bb.11:
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:  .LBB34_12:
+; SSE2-NEXT:    maxss %xmm3, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    shll $16, %r14d
+; SSE2-NEXT:    movd %r14d, %xmm1
+; SSE2-NEXT:    shll $16, %r15d
+; SSE2-NEXT:    movd %r15d, %xmm3
+; SSE2-NEXT:    testl %r14d, %r14d
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    js .LBB34_14
+; SSE2-NEXT:  # %bb.13:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:  .LBB34_14:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm2, %xmm4
+; SSE2-NEXT:    js .LBB34_16
+; SSE2-NEXT:  # %bb.15:
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:  .LBB34_16:
+; SSE2-NEXT:    maxss %xmm3, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; SSE2-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    addq $56, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 40
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    .cfi_def_cfa_offset 32
+; SSE2-NEXT:    popq %r14
+; SSE2-NEXT:    .cfi_def_cfa_offset 24
+; SSE2-NEXT:    popq %r15
+; SSE2-NEXT:    .cfi_def_cfa_offset 16
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    .cfi_def_cfa_offset 8
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_v4bf16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    pushq %rbp
+; AVX1-NEXT:    .cfi_def_cfa_offset 16
+; AVX1-NEXT:    pushq %r15
+; AVX1-NEXT:    .cfi_def_cfa_offset 24
+; AVX1-NEXT:    pushq %r14
+; AVX1-NEXT:    .cfi_def_cfa_offset 32
+; AVX1-NEXT:    pushq %r13
+; AVX1-NEXT:    .cfi_def_cfa_offset 40
+; AVX1-NEXT:    pushq %r12
+; AVX1-NEXT:    .cfi_def_cfa_offset 48
+; AVX1-NEXT:    pushq %rbx
+; AVX1-NEXT:    .cfi_def_cfa_offset 56
+; AVX1-NEXT:    subq $56, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 112
+; AVX1-NEXT:    .cfi_offset %rbx, -56
+; AVX1-NEXT:    .cfi_offset %r12, -48
+; AVX1-NEXT:    .cfi_offset %r13, -40
+; AVX1-NEXT:    .cfi_offset %r14, -32
+; AVX1-NEXT:    .cfi_offset %r15, -24
+; AVX1-NEXT:    .cfi_offset %rbp, -16
+; AVX1-NEXT:    vpsrlq $48, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $48, %xmm1, %xmm3
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpextrw $0, %xmm4, %ebx
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpextrw $0, %xmm4, %ebp
+; AVX1-NEXT:    vpextrw $0, %xmm0, %r12d
+; AVX1-NEXT:    vpextrw $0, %xmm1, %r13d
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrw $0, %xmm0, %eax
+; AVX1-NEXT:    vpsrld $16, %xmm1, %xmm0
+; AVX1-NEXT:    vpextrw $0, %xmm0, %ecx
+; AVX1-NEXT:    shll $16, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm4
+; AVX1-NEXT:    js .LBB34_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmovdqa %xmm4, %xmm1
+; AVX1-NEXT:    jmp .LBB34_3
+; AVX1-NEXT:  .LBB34_1:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa %xmm4, %xmm0
+; AVX1-NEXT:  .LBB34_3:
+; AVX1-NEXT:    vpextrw $0, %xmm2, %r14d
+; AVX1-NEXT:    vpextrw $0, %xmm3, %r15d
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    callq __truncsfbf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    shll $16, %r13d
+; AVX1-NEXT:    vmovd %r13d, %xmm0
+; AVX1-NEXT:    shll $16, %r12d
+; AVX1-NEXT:    vmovd %r12d, %xmm2
+; AVX1-NEXT:    js .LBB34_4
+; AVX1-NEXT:  # %bb.5:
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm1
+; AVX1-NEXT:    jmp .LBB34_6
+; AVX1-NEXT:  .LBB34_4:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX1-NEXT:  .LBB34_6:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    callq __truncsfbf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    shll $16, %ebp
+; AVX1-NEXT:    vmovd %ebp, %xmm0
+; AVX1-NEXT:    shll $16, %ebx
+; AVX1-NEXT:    vmovd %ebx, %xmm2
+; AVX1-NEXT:    js .LBB34_7
+; AVX1-NEXT:  # %bb.8:
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm1
+; AVX1-NEXT:    jmp .LBB34_9
+; AVX1-NEXT:  .LBB34_7:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX1-NEXT:  .LBB34_9:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    callq __truncsfbf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    shll $16, %r15d
+; AVX1-NEXT:    vmovd %r15d, %xmm0
+; AVX1-NEXT:    shll $16, %r14d
+; AVX1-NEXT:    vmovd %r14d, %xmm2
+; AVX1-NEXT:    js .LBB34_10
+; AVX1-NEXT:  # %bb.11:
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm1
+; AVX1-NEXT:    jmp .LBB34_12
+; AVX1-NEXT:  .LBB34_10:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX1-NEXT:  .LBB34_12:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    callq __truncsfbf2@PLT
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; AVX1-NEXT:    addq $56, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 56
+; AVX1-NEXT:    popq %rbx
+; AVX1-NEXT:    .cfi_def_cfa_offset 48
+; AVX1-NEXT:    popq %r12
+; AVX1-NEXT:    .cfi_def_cfa_offset 40
+; AVX1-NEXT:    popq %r13
+; AVX1-NEXT:    .cfi_def_cfa_offset 32
+; AVX1-NEXT:    popq %r14
+; AVX1-NEXT:    .cfi_def_cfa_offset 24
+; AVX1-NEXT:    popq %r15
+; AVX1-NEXT:    .cfi_def_cfa_offset 16
+; AVX1-NEXT:    popq %rbp
+; AVX1-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fmaximum_v4bf16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    pushq %rbp
+; AVX512-NEXT:    .cfi_def_cfa_offset 16
+; AVX512-NEXT:    pushq %r15
+; AVX512-NEXT:    .cfi_def_cfa_offset 24
+; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    .cfi_def_cfa_offset 32
+; AVX512-NEXT:    pushq %r13
+; AVX512-NEXT:    .cfi_def_cfa_offset 40
+; AVX512-NEXT:    pushq %r12
+; AVX512-NEXT:    .cfi_def_cfa_offset 48
+; AVX512-NEXT:    pushq %rbx
+; AVX512-NEXT:    .cfi_def_cfa_offset 56
+; AVX512-NEXT:    pushq %rax
+; AVX512-NEXT:    .cfi_def_cfa_offset 64
+; AVX512-NEXT:    .cfi_offset %rbx, -56
+; AVX512-NEXT:    .cfi_offset %r12, -48
+; AVX512-NEXT:    .cfi_offset %r13, -40
+; AVX512-NEXT:    .cfi_offset %r14, -32
+; AVX512-NEXT:    .cfi_offset %r15, -24
+; AVX512-NEXT:    .cfi_offset %rbp, -16
+; AVX512-NEXT:    vmovq %xmm1, %r13
+; AVX512-NEXT:    movq %r13, %rbx
+; AVX512-NEXT:    shrq $32, %rbx
+; AVX512-NEXT:    vmovq %xmm0, %rbp
+; AVX512-NEXT:    movq %rbp, %r14
+; AVX512-NEXT:    shrq $32, %r14
+; AVX512-NEXT:    movq %r13, %r15
+; AVX512-NEXT:    shrq $48, %r15
+; AVX512-NEXT:    movq %rbp, %r12
+; AVX512-NEXT:    shrq $48, %r12
+; AVX512-NEXT:    movl %ebp, %eax
+; AVX512-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
+; AVX512-NEXT:    sets %cl
+; AVX512-NEXT:    kmovw %ecx, %k1
+; AVX512-NEXT:    movl %r13d, %ecx
+; AVX512-NEXT:    andl $-65536, %ecx # imm = 0xFFFF0000
+; AVX512-NEXT:    vmovd %ecx, %xmm1
+; AVX512-NEXT:    vmovd %eax, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    callq __truncsfbf2@PLT
+; AVX512-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; AVX512-NEXT:    shll $16, %ebp
+; AVX512-NEXT:    sets %al
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    shll $16, %r13d
+; AVX512-NEXT:    vmovd %r13d, %xmm1
+; AVX512-NEXT:    vmovd %ebp, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    callq __truncsfbf2@PLT
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsp)
+; AVX512-NEXT:    shll $16, %r12d
+; AVX512-NEXT:    sets %al
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    shll $16, %r15d
+; AVX512-NEXT:    vmovd %r15d, %xmm1
+; AVX512-NEXT:    vmovd %r12d, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    callq __truncsfbf2@PLT
+; AVX512-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; AVX512-NEXT:    shll $16, %r14d
+; AVX512-NEXT:    sets %al
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    shll $16, %ebx
+; AVX512-NEXT:    vmovd %ebx, %xmm1
+; AVX512-NEXT:    vmovd %r14d, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    callq __truncsfbf2@PLT
+; AVX512-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; AVX512-NEXT:    vmovaps (%rsp), %xmm0
+; AVX512-NEXT:    addq $8, %rsp
+; AVX512-NEXT:    .cfi_def_cfa_offset 56
+; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    .cfi_def_cfa_offset 48
+; AVX512-NEXT:    popq %r12
+; AVX512-NEXT:    .cfi_def_cfa_offset 40
+; AVX512-NEXT:    popq %r13
+; AVX512-NEXT:    .cfi_def_cfa_offset 32
+; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    .cfi_def_cfa_offset 24
+; AVX512-NEXT:    popq %r15
+; AVX512-NEXT:    .cfi_def_cfa_offset 16
+; AVX512-NEXT:    popq %rbp
+; AVX512-NEXT:    .cfi_def_cfa_offset 8
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_v4bf16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 20
+; X86-NEXT:    subl $68, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 88
+; X86-NEXT:    .cfi_offset %esi, -20
+; X86-NEXT:    .cfi_offset %edi, -16
+; X86-NEXT:    .cfi_offset %ebx, -12
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    vpsrlq $48, %xmm0, %xmm2
+; X86-NEXT:    vpsrlq $48, %xmm1, %xmm3
+; X86-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; X86-NEXT:    vpextrw $0, %xmm4, %esi
+; X86-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; X86-NEXT:    vpextrw $0, %xmm4, %ebx
+; X86-NEXT:    vpextrw $0, %xmm0, %eax
+; X86-NEXT:    vpextrw $0, %xmm1, %ecx
+; X86-NEXT:    vpsrld $16, %xmm0, %xmm0
+; X86-NEXT:    vpextrw $0, %xmm0, %edx
+; X86-NEXT:    vpsrld $16, %xmm1, %xmm0
+; X86-NEXT:    vpextrw $0, %xmm0, %edi
+; X86-NEXT:    shll $16, %edi
+; X86-NEXT:    vmovd %edi, %xmm0
+; X86-NEXT:    shll $16, %edx
+; X86-NEXT:    vmovd %edx, %xmm4
+; X86-NEXT:    js .LBB34_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovdqa %xmm4, %xmm1
+; X86-NEXT:    jmp .LBB34_3
+; X86-NEXT:  .LBB34_1:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm4, %xmm0
+; X86-NEXT:  .LBB34_3:
+; X86-NEXT:    vpextrw $0, %xmm2, %edi
+; X86-NEXT:    vpextrw $0, %xmm3, %ebp
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    shll $16, %ecx
+; X86-NEXT:    vmovd %ecx, %xmm0
+; X86-NEXT:    shll $16, %eax
+; X86-NEXT:    vmovd %eax, %xmm2
+; X86-NEXT:    js .LBB34_4
+; X86-NEXT:  # %bb.5:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB34_6
+; X86-NEXT:  .LBB34_4:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB34_6:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfbf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    shll $16, %ebx
+; X86-NEXT:    vmovd %ebx, %xmm0
+; X86-NEXT:    shll $16, %esi
+; X86-NEXT:    vmovd %esi, %xmm2
+; X86-NEXT:    js .LBB34_7
+; X86-NEXT:  # %bb.8:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB34_9
+; X86-NEXT:  .LBB34_7:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB34_9:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfbf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    shll $16, %ebp
+; X86-NEXT:    vmovd %ebp, %xmm0
+; X86-NEXT:    shll $16, %edi
+; X86-NEXT:    vmovd %edi, %xmm2
+; X86-NEXT:    js .LBB34_10
+; X86-NEXT:  # %bb.11:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB34_12
+; X86-NEXT:  .LBB34_10:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB34_12:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfbf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovd %xmm0, (%esp)
+; X86-NEXT:    calll __truncsfbf2
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; X86-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; X86-NEXT:    addl $68, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 20
+; X86-NEXT:    popl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    popl %edi
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+  %r = call <4 x bfloat> @llvm.maximum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
+  ret <4 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/X86/fold-loop-of-urem.ll b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
new file mode 100644
index 000000000000000..aad2e0dd7bd248a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
@@ -0,0 +1,1159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+declare void @use.i32(i32)
+declare void @use.2xi64(<2 x i64>)
+declare void @do_stuff0()
+declare void @do_stuff1()
+declare i1 @get.i1()
+declare i32 @get.i32()
+
+define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB0_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB0_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_nested2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB1_8
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    jmp .LBB1_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_5: # %for.body1
+; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:  .LBB1_6: # %for.body.tail
+; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    je .LBB1_7
+; CHECK-NEXT:  .LBB1_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    je .LBB1_6
+; CHECK-NEXT:  # %bb.3: # %for.body0
+; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB1_5
+; CHECK-NEXT:  # %bb.4: # %for.body2
+; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB1_5
+; CHECK-NEXT:    jmp .LBB1_6
+; CHECK-NEXT:  .LBB1_7:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB1_8: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body.tail ], [ 0, %entry ]
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body.tail
+for.body0:
+  %cond1 = call i1 @get.i1()
+  br i1 %cond1, label %for.body1, label %for.body2
+for.body2:
+  %cond2 = call i1 @get.i1()
+  br i1 %cond2, label %for.body1, label %for.body.tail
+for.body1:
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  br label %for.body.tail
+for.body.tail:
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr3(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_bad_incr3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB2_9
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    jmp .LBB2_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB2_6: # %for.body1
+; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:  .LBB2_7: # %for.body.tail
+; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB2_8
+; CHECK-NEXT:  .LBB2_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    je .LBB2_5
+; CHECK-NEXT:  # %bb.3: # %for.body0
+; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    movl %eax, %r14d
+; CHECK-NEXT:    callq get.i32@PLT
+; CHECK-NEXT:    testb $1, %r14b
+; CHECK-NEXT:    je .LBB2_7
+; CHECK-NEXT:  # %bb.4: # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    movl %eax, %ebp
+; CHECK-NEXT:    incl %ebp
+; CHECK-NEXT:    jmp .LBB2_6
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB2_5: # %for.body2
+; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    xorl %ebp, %ebp
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB2_6
+; CHECK-NEXT:    jmp .LBB2_7
+; CHECK-NEXT:  .LBB2_8:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB2_9: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body2
+for.body0:
+  %cond1 = call i1 @get.i1()
+  %val = call i32 @get.i32()
+  %inc = add nuw i32 %val, 1
+  br i1 %cond1, label %for.body1, label %for.body.tail
+for.body2:
+  %cond2 = call i1 @get.i1()
+  br i1 %cond2, label %for.body1, label %for.body.tail
+for.body1:
+  %i.04 = phi i32 [ %inc, %for.body0], [ 0, %for.body2 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  br label %for.body.tail
+for.body.tail:
+  %exitcond.not = call i1 @get.i1()
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_vec:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movq %xmm0, %rbx
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; CHECK-NEXT:    movq %xmm0, %r14
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB3_1: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rbx
+; CHECK-NEXT:    movq %rdx, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %r14
+; CHECK-NEXT:    movq %rdx, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    callq use.2xi64@PLT
+; CHECK-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    psubq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    je .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    retq
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi <2 x i64> [ %inc, %for.body ], [ zeroinitializer, %entry ]
+  %rem = urem <2 x i64> %i.04, %rem_amt
+  tail call void @use.2xi64(<2 x i64> %rem)
+  %inc = add nuw <2 x i64> %i.04, <i64 1, i64 1>
+  %exitcond.not = call i1 @get.i1()
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_bad_incr:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB4_6
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    jmp .LBB4_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB4_4: # %for.body.tail
+; CHECK-NEXT:    # in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %ebp, %r14d
+; CHECK-NEXT:    je .LBB4_5
+; CHECK-NEXT:  .LBB4_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    je .LBB4_4
+; CHECK-NEXT:  # %bb.3: # %for.body0
+; CHECK-NEXT:    # in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    callq get.i32@PLT
+; CHECK-NEXT:    movl %eax, %r14d
+; CHECK-NEXT:    jmp .LBB4_4
+; CHECK-NEXT:  .LBB4_5:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB4_6: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.03 = phi i32 [ %inc, %for.body.tail ], [ 0, %entry ]
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body.tail
+for.body0:
+  %some_val = call i32 @get.i32()
+  br label %for.body.tail
+
+for.body.tail:
+  %i.04 = phi i32 [ %i.03, %for.body ], [ %some_val, %for.body0 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_second_acc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $2, %edi
+; CHECK-NEXT:    jb .LBB5_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    movl $1, %r15d
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB5_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    addl $2, %r15d
+; CHECK-NEXT:    cmpl %ebp, %r15d
+; CHECK-NEXT:    jbe .LBB5_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB5_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ult i32 %N, 2
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %i.05 = phi i32 [ %inc2, %for.body ], [ 1, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %inc2 = add nuw i32 %i.05, 2
+  %exitcond.not = icmp ugt i32 %inc2, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_srem(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_srem:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB6_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB6_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB6_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB6_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = srem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_missing_nuw(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_missing_nuw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB7_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB7_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB7_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB7_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nsw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr2(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_bad_incr2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB8_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB8_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    addl $2, %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB8_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB8_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 2
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_non_zero_entry4(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_non_zero_entry4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB9_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    movl $4, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB9_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB9_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB9_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 4, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_skip_const_rem_amt(i32 %N) nounwind {
+; CHECK-LABEL: simple_urem_skip_const_rem_amt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB10_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    addl $-4, %ebx
+; CHECK-NEXT:    movl $4, %ebp
+; CHECK-NEXT:    movl $2938661835, %r14d # imm = 0xAF286BCB
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB10_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:    imulq %r14, %rax
+; CHECK-NEXT:    shrq $32, %rax
+; CHECK-NEXT:    movl %ebp, %ecx
+; CHECK-NEXT:    subl %eax, %ecx
+; CHECK-NEXT:    shrl %ecx
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    shrl $4, %ecx
+; CHECK-NEXT:    leal (%rcx,%rcx,8), %eax
+; CHECK-NEXT:    leal (%rcx,%rax,2), %eax
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    subl %eax, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %ebp
+; CHECK-NEXT:    decl %ebx
+; CHECK-NEXT:    jne .LBB10_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB10_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 4, %entry ]
+  %rem = urem i32 %i.04, 19
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_no_preheader_non_canonical(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_no_preheader_non_canonical:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB11_1
+; CHECK-NEXT:  # %bb.2: # %for.body1
+; CHECK-NEXT:    movl $1, %r14d
+; CHECK-NEXT:    jmp .LBB11_3
+; CHECK-NEXT:  .LBB11_1:
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB11_3: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB11_3
+; CHECK-NEXT:  # %bb.4: # %for.cond.cleanup
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.body0, label %for.body1
+
+for.cond.cleanup:
+  ret void
+
+for.body0:
+  br label %for.body
+
+for.body1:
+  br label %for.body
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %for.body0 ], [ 1, %for.body1 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_multi_latch_non_canonical:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB12_6
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    decl %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    jmp .LBB12_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB12_3: # %for.body.backedge
+; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:  .LBB12_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    movl %eax, %r15d
+; CHECK-NEXT:    callq do_stuff0@PLT
+; CHECK-NEXT:    testb $1, %r15b
+; CHECK-NEXT:    je .LBB12_3
+; CHECK-NEXT:  # %bb.4: # %for.body0
+; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
+; CHECK-NEXT:    callq do_stuff1@PLT
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB12_3
+; CHECK-NEXT:  # %bb.5:
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB12_6: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %inc, %for.body0 ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %cond = call i1 @get.i1()
+  call void @do_stuff0()
+  br i1 %cond, label %for.body0, label %for.body
+for.body0:
+  call void @do_stuff1()
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_loop(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_bad_loop:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    callq get.i32@PLT
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    # implicit-def: $r14d
+; CHECK-NEXT:    jne .LBB13_4
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:  .LBB13_2: # %for.cond
+; CHECK-NEXT:    cmpl %ebp, %r14d
+; CHECK-NEXT:    jae .LBB13_5
+; CHECK-NEXT:  # %bb.3: # %for.body
+; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    xorl $1, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:  .LBB13_4: # %halfway
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    jmp .LBB13_2
+; CHECK-NEXT:  .LBB13_5: # %for.end
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %call = call i32 @get.i32()
+  %tobool.not = icmp eq i32 %call, 0
+  br i1 %tobool.not, label %for.cond, label %halfway
+
+for.cond:
+  %i.0 = phi i32 [ %inc, %halfway ], [ 0, %entry ]
+  %cmp = icmp ult i32 %i.0, %N
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %xor = xor i32 %i.0, 1
+  call void @use.i32(i32 %xor)
+  br label %halfway
+
+halfway:
+  %i.1 = phi i32 [ poison, %entry ], [ %i.0, %for.body ]
+  %rem = urem i32 %i.1, %rem_amt
+  call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.1, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+define void @simple_urem_fail_intermediate_inc(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_intermediate_inc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB14_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    movl $1, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB14_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB14_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB14_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %inc2 = add nuw i32 %i.04, 1
+  %rem = urem i32 %inc2, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @weird_loop(i64 %sub.ptr.div.i56) personality ptr null {
+; CHECK-LABEL: weird_loop:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB15_1: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jmp .LBB15_1
+entry:
+  br label %for.preheader
+
+for.preheader:
+  %i57.0540.us = phi i64 [ 0, %entry ], [ %add74.us, %for.body ]
+  %add74.us = add nuw i64 %i57.0540.us, 1
+  br label %for.body
+
+for.body:
+  %rem.us = urem i64 %i57.0540.us, %sub.ptr.div.i56
+  br i1 false, label %for.preheader, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $3, %edi
+; CHECK-NEXT:    jb .LBB16_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    movl $2, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB16_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB16_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB16_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add(i32 %N, i32 %rem_amt_in) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_add:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $3, %edi
+; CHECK-NEXT:    jb .LBB17_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    orl $16, %ebx
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    movl $7, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB17_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $5, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB17_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB17_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %rem_amt = or i32 %rem_amt_in, 16
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add nuw i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw(i32 %N, i32 %rem_amt_in) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $3, %edi
+; CHECK-NEXT:    jb .LBB18_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    orl $16, %ebx
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    movl $7, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB18_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $5, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB18_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB18_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %rem_amt = or i32 %rem_amt_in, 16
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $3, %edi
+; CHECK-NEXT:    jb .LBB19_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    movl $7, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB19_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $5, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB19_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB19_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add nuw i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_sub(i32 %N, i32 %rem_amt, i32 %start) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_sub:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    subl %edx, %ebp
+; CHECK-NEXT:    jbe .LBB20_3
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB20_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB20_2
+; CHECK-NEXT:  .LBB20_3: # %for.cond.cleanup
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ule i32 %N, %start
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %start, %entry ]
+  %i_with_off = sub nuw i32 %i.04, %start
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_sub_no_simplfy(i32 %N, i32 %rem_amt, i32 %start) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_sub_no_simplfy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl %edx, %edi
+; CHECK-NEXT:    jbe .LBB21_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %edx, %r15d
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    addl $-2, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB21_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $-2, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB21_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB21_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ule i32 %N, %start
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %start, %entry ]
+  %i_with_off = sub nuw i32 %i.04, 2
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/CodeGen/X86/huge-stack.ll b/llvm/test/CodeGen/X86/huge-stack.ll
index 920033ba1182c3f..41b8a0141b63d8b 100644
--- a/llvm/test/CodeGen/X86/huge-stack.ll
+++ b/llvm/test/CodeGen/X86/huge-stack.ll
@@ -5,20 +5,70 @@
 define void @foo() unnamed_addr #0 {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movabsq $8589934462, %rax # imm = 0x1FFFFFF7E
+; CHECK-NEXT:    movabsq $8589934472, %rax # imm = 0x1FFFFFF88
 ; CHECK-NEXT:    subq %rax, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8589934470
-; CHECK-NEXT:    movb $42, -129(%rsp)
-; CHECK-NEXT:    movb $43, -128(%rsp)
-; CHECK-NEXT:    movabsq $8589934462, %rax # imm = 0x1FFFFFF7E
+; CHECK-NEXT:    .cfi_def_cfa_offset 8589934480
+; CHECK-NEXT:    movabsq $4294967177, %rax # imm = 0xFFFFFF89
+; CHECK-NEXT:    movb $42, (%rsp,%rax)
+; CHECK-NEXT:    movb $43, -118(%rsp)
+; CHECK-NEXT:    movabsq $8589934472, %rax # imm = 0x1FFFFFF88
 ; CHECK-NEXT:    addq %rax, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
-  %1 = alloca %large, align 1
-  %2 = alloca %large, align 1
-  %3 = getelementptr inbounds %large, ptr %1, i64 0, i64 0
-  store i8 42, ptr %3, align 1
-  %4 = getelementptr inbounds %large, ptr %2, i64 0, i64 0
-  store i8 43, ptr %4, align 1
+  %large1 = alloca %large, align 1
+  %large2 = alloca %large, align 1
+  %ptrLarge1 = getelementptr inbounds %large, ptr %large1, i64 0, i64 0
+  store i8 42, ptr %ptrLarge1, align 1
+  %ptrLarge2 = getelementptr inbounds %large, ptr %large2, i64 0, i64 0
+  store i8 43, ptr %ptrLarge2, align 1
   ret void
 }
+
+declare ptr @baz(ptr, ptr, ptr, ptr)
+
+define ptr @scavenge_spill() unnamed_addr #0 {
+; CHECK-LABEL: scavenge_spill:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $25769803816, %rax # imm = 0x600000028
+; CHECK-NEXT:    subq %rax, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 25769803824
+; CHECK-NEXT:    movabsq $21474836521, %rax # imm = 0x500000029
+; CHECK-NEXT:    leaq (%rsp,%rax), %rdi
+; CHECK-NEXT:    movabsq $17179869226, %rax # imm = 0x40000002A
+; CHECK-NEXT:    leaq (%rsp,%rax), %rsi
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movabsq $12884901931, %rax # imm = 0x30000002B
+; CHECK-NEXT:    leaq (%rsp,%rax), %rdx
+; CHECK-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movabsq $8589934636, %rax # imm = 0x20000002C
+; CHECK-NEXT:    leaq (%rsp,%rax), %rcx
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    callq baz@PLT
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    leaq 46(%rsp), %rdi
+; CHECK-NEXT:    callq baz@PLT
+; CHECK-NEXT:    # kill: def $rcx killed $rax
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    movabsq $25769803816, %rcx # imm = 0x600000028
+; CHECK-NEXT:    addq %rcx, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %large1 = alloca %large, align 1
+  %ptrLarge1 = getelementptr inbounds %large, ptr %large1, i64 0, i64 0
+  %large2 = alloca %large, align 1
+  %ptrLarge2 = getelementptr inbounds %large, ptr %large2, i64 0, i64 0
+  %large3 = alloca %large, align 1
+  %ptrLarge3 = getelementptr inbounds %large, ptr %large3, i64 0, i64 0
+  %large4 = alloca %large, align 1
+  %ptrLarge4 = getelementptr inbounds %large, ptr %large4, i64 0, i64 0
+  %large5 = alloca %large, align 1
+  %ptrLarge5 = getelementptr inbounds %large, ptr %large5, i64 0, i64 0
+  %ret1 = call ptr @baz(ptr %ptrLarge1, ptr %ptrLarge2, ptr %ptrLarge3, ptr %ptrLarge4)
+  %large6 = alloca %large, align 1
+  %ptrLarge6 = getelementptr inbounds %large, ptr %large6, i64 0, i64 0
+  %ret2 = call ptr @baz(ptr %ptrLarge6, ptr %ptrLarge2, ptr %ptrLarge3, ptr %ptrLarge4)
+  ret ptr %ret1
+}
diff --git a/llvm/test/CodeGen/X86/pr50609.ll b/llvm/test/CodeGen/X86/pr50609.ll
index ea85c312f38bd1c..3e39c567d79e6df 100644
--- a/llvm/test/CodeGen/X86/pr50609.ll
+++ b/llvm/test/CodeGen/X86/pr50609.ll
@@ -12,7 +12,7 @@ define void @PR50609(ptr noalias nocapture %RET, ptr noalias %aFOO, <16 x i32> %
 ; CHECK-NEXT:    vpsrld $30, %xmm3, %xmm3
 ; CHECK-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; CHECK-NEXT:    vpsrad $2, %xmm2, %xmm2
-; CHECK-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; CHECK-NEXT:    vcvtdq2ps %xmm2, %xmm2
 ; CHECK-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
 ; CHECK-NEXT:    vmaskmovps %ymm2, %ymm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/shuffle-half.ll b/llvm/test/CodeGen/X86/shuffle-half.ll
index 291fe841043ed42..001db2c7cecae1f 100644
--- a/llvm/test/CodeGen/X86/shuffle-half.ll
+++ b/llvm/test/CodeGen/X86/shuffle-half.ll
@@ -4,305 +4,13 @@
 define <32 x half> @dump_vec() {
 ; CHECK-LABEL: dump_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %cond.load
-; CHECK-NEXT:    vpinsrw $0, (%rax), %xmm0, %xmm0
-; CHECK-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]
-; CHECK-NEXT:    vpand %ymm0, %ymm1, %ymm0
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; CHECK-NEXT:  .LBB0_2: # %else
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_4
-; CHECK-NEXT:  # %bb.3: # %cond.load1
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_4: # %else2
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_6
-; CHECK-NEXT:  # %bb.5: # %cond.load4
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2],xmm0[3,4,5,6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_6: # %else5
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_8
-; CHECK-NEXT:  # %bb.7: # %cond.load7
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_8: # %else8
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_10
-; CHECK-NEXT:  # %bb.9: # %cond.load10
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_10: # %else11
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_12
-; CHECK-NEXT:  # %bb.11: # %cond.load13
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3,4],xmm1[5],xmm0[6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_12: # %else14
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_14
-; CHECK-NEXT:  # %bb.13: # %cond.load16
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5],xmm1[6],xmm0[7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_14: # %else17
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_16
-; CHECK-NEXT:  # %bb.15: # %cond.load19
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,6],xmm1[7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_16: # %else20
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_18
-; CHECK-NEXT:  # %bb.17: # %cond.load22
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_18: # %else23
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_20
-; CHECK-NEXT:  # %bb.19: # %cond.load25
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7,8],ymm1[9],ymm0[10,11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_20: # %else26
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_22
-; CHECK-NEXT:  # %bb.21: # %cond.load28
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4,5,6,7,8,9],ymm1[10],ymm0[11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_22: # %else29
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_24
-; CHECK-NEXT:  # %bb.23: # %cond.load31
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7,8,9,10],ymm1[11],ymm0[12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_24: # %else32
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_26
-; CHECK-NEXT:  # %bb.25: # %cond.load34
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_26: # %else35
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_28
-; CHECK-NEXT:  # %bb.27: # %cond.load37
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7,8,9,10,11,12],ymm1[13],ymm0[14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_28: # %else38
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_30
-; CHECK-NEXT:  # %bb.29: # %cond.load40
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7,8,9,10,11,12,13],ymm1[14],ymm0[15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_30: # %else41
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_32
-; CHECK-NEXT:  # %bb.31: # %cond.load43
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_32: # %else44
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_34
-; CHECK-NEXT:  # %bb.33: # %cond.load46
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_34: # %else47
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_36
-; CHECK-NEXT:  # %bb.35: # %cond.load49
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3,4,5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_36: # %else50
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_38
-; CHECK-NEXT:  # %bb.37: # %cond.load52
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3,4,5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_38: # %else53
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_40
-; CHECK-NEXT:  # %bb.39: # %cond.load55
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_40: # %else56
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_42
-; CHECK-NEXT:  # %bb.41: # %cond.load58
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4],xmm2[5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_42: # %else59
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_44
-; CHECK-NEXT:  # %bb.43: # %cond.load61
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4],xmm1[5],xmm2[6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_44: # %else62
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_46
-; CHECK-NEXT:  # %bb.45: # %cond.load64
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5],xmm1[6],xmm2[7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_46: # %else65
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_48
-; CHECK-NEXT:  # %bb.47: # %cond.load67
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5,6],xmm1[7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_48: # %else68
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_50
-; CHECK-NEXT:  # %bb.49: # %cond.load70
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_50: # %else71
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_52
-; CHECK-NEXT:  # %bb.51: # %cond.load73
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4,5,6,7,8],ymm1[9],ymm2[10,11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_52: # %else74
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_54
-; CHECK-NEXT:  # %bb.53: # %cond.load76
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4,5,6,7,8,9],ymm1[10],ymm2[11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_54: # %else77
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_56
-; CHECK-NEXT:  # %bb.55: # %cond.load79
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6,7,8,9,10],ymm1[11],ymm2[12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_56: # %else80
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_58
-; CHECK-NEXT:  # %bb.57: # %cond.load82
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4],ymm2[5,6,7,8,9,10,11],ymm1[12],ymm2[13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_58: # %else83
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_60
-; CHECK-NEXT:  # %bb.59: # %cond.load85
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7,8,9,10,11,12],ymm1[13],ymm2[14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_60: # %else86
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_62
-; CHECK-NEXT:  # %bb.61: # %cond.load88
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6],ymm2[7,8,9,10,11,12,13],ymm1[14],ymm2[15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_62: # %else89
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_64
-; CHECK-NEXT:  # %bb.63: # %cond.load91
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7],ymm2[8,9,10,11,12,13,14],ymm1[15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_64: # %else92
+; CHECK-NEXT:    vmovups (%rax), %zmm0
+; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    retq
   %1 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr poison, i32 2, <32 x i1> poison, <32 x half> <half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0>)
   ret <32 x half> %1
diff --git a/llvm/test/CodeGen/X86/unpredictable-brcond.ll b/llvm/test/CodeGen/X86/unpredictable-brcond.ll
index 6c894ea8277679c..12411f1c49f2dbb 100644
--- a/llvm/test/CodeGen/X86/unpredictable-brcond.ll
+++ b/llvm/test/CodeGen/X86/unpredictable-brcond.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-; Currently, unpredictable metadata on conditional branches is lost during CodeGen.
+; Make sure MIR generated for conditional branch with unpredictable metadata has unpredictable flag.
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -stop-after=finalize-isel < %s | FileCheck %s
 
 define void @cond_branch_1(i1 %cond) {
@@ -11,7 +11,7 @@ define void @cond_branch_1(i1 %cond) {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $edi
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit
   ; CHECK-NEXT:   TEST8ri killed [[COPY1]], 1, implicit-def $eflags
-  ; CHECK-NEXT:   JCC_1 %bb.2, 4, implicit $eflags
+  ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 4, implicit $eflags
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.then:
@@ -51,7 +51,7 @@ define void @cond_branch_2(double %a, double %b, i32 %c, i32 %d) nounwind {
   ; CHECK-NEXT:   [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
   ; CHECK-NEXT:   [[OR8rr:%[0-9]+]]:gr8 = OR8rr [[SETCCr]], killed [[SETCCr1]], implicit-def dead $eflags
   ; CHECK-NEXT:   TEST8rr [[OR8rr]], [[OR8rr]], implicit-def $eflags
-  ; CHECK-NEXT:   JCC_1 %bb.2, 5, implicit $eflags
+  ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 5, implicit $eflags
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.true:
@@ -89,8 +89,8 @@ define void @isint_branch(double %d) nounwind {
   ; CHECK-NEXT:   [[CVTDQ2PDrr:%[0-9]+]]:vr128 = CVTDQ2PDrr killed [[CVTTPD2DQrr]]
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:fr64 = COPY [[CVTDQ2PDrr]]
   ; CHECK-NEXT:   nofpexcept UCOMISDrr [[COPY]], killed [[COPY2]], implicit-def $eflags, implicit $mxcsr
-  ; CHECK-NEXT:   JCC_1 %bb.2, 5, implicit $eflags
-  ; CHECK-NEXT:   JCC_1 %bb.2, 10, implicit $eflags
+  ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 5, implicit $eflags
+  ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 10, implicit $eflags
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.true:
diff --git a/llvm/test/CodeGen/X86/vec_int_to_fp.ll b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
index e26de4be7066fa5..af841cf38b24aec 100644
--- a/llvm/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
@@ -1478,32 +1478,11 @@ define <4 x float> @sitofp_8i16_to_4f32(<8 x i16> %a) {
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sitofp_8i16_to_4f32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: sitofp_8i16_to_4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: sitofp_8i16_to_4f32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: sitofp_8i16_to_4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
   %cvt = sitofp <8 x i16> %a to <8 x float>
   %shuf = shufflevector <8 x float> %cvt, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %shuf
@@ -1549,32 +1528,11 @@ define <4 x float> @sitofp_16i8_to_4f32(<16 x i8> %a) {
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sitofp_16i8_to_4f32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: sitofp_16i8_to_4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: sitofp_16i8_to_4f32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512-NEXT:    vcvtdq2ps %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: sitofp_16i8_to_4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
   %cvt = sitofp <16 x i8> %a to <16 x float>
   %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %shuf
@@ -2354,32 +2312,11 @@ define <4 x float> @uitofp_8i16_to_4f32(<8 x i16> %a) {
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: uitofp_8i16_to_4f32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: uitofp_8i16_to_4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: uitofp_8i16_to_4f32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: uitofp_8i16_to_4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
   %cvt = uitofp <8 x i16> %a to <8 x float>
   %shuf = shufflevector <8 x float> %cvt, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %shuf
@@ -2425,32 +2362,11 @@ define <4 x float> @uitofp_16i8_to_4f32(<16 x i8> %a) {
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: uitofp_16i8_to_4f32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: uitofp_16i8_to_4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: uitofp_16i8_to_4f32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512-NEXT:    vcvtdq2ps %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: uitofp_16i8_to_4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
   %cvt = uitofp <16 x i8> %a to <16 x float>
   %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %shuf
diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll
index e87814ebb1dbe48..ef0f3f3e816dfa3 100644
--- a/llvm/test/CodeGen/X86/vector-half-conversions.ll
+++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll
@@ -4990,6 +4990,7 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind {
   ret <4 x i32> %ext
 }
 
+; PR83402
 define <4 x i32> @fptosi_4f16_to_4i32(<4 x half> %a) nounwind {
 ; AVX-LABEL: fptosi_4f16_to_4i32:
 ; AVX:       # %bb.0:
@@ -5024,16 +5025,14 @@ define <4 x i32> @fptosi_4f16_to_4i32(<4 x half> %a) nounwind {
 ;
 ; F16C-LABEL: fptosi_4f16_to_4i32:
 ; F16C:       # %bb.0:
-; F16C-NEXT:    vcvtph2ps %xmm0, %ymm0
+; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; F16C-NEXT:    vcvttps2dq %xmm0, %xmm0
-; F16C-NEXT:    vzeroupper
 ; F16C-NEXT:    retq
 ;
 ; AVX512-LABEL: fptosi_4f16_to_4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %cvt = fptosi <4 x half> %a to <4 x i32>
   ret <4 x i32> %cvt
@@ -5213,13 +5212,12 @@ define <4 x i32> @fptoui_4f16_to_4i32(<4 x half> %a) nounwind {
 ;
 ; F16C-LABEL: fptoui_4f16_to_4i32:
 ; F16C:       # %bb.0:
-; F16C-NEXT:    vcvtph2ps %xmm0, %ymm0
-; F16C-NEXT:    vcvttps2dq %ymm0, %ymm1
-; F16C-NEXT:    vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; F16C-NEXT:    vcvttps2dq %ymm0, %ymm0
+; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
+; F16C-NEXT:    vcvttps2dq %xmm0, %xmm1
+; F16C-NEXT:    vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; F16C-NEXT:    vcvttps2dq %xmm0, %xmm0
 ; F16C-NEXT:    vorps %xmm0, %xmm1, %xmm0
 ; F16C-NEXT:    vblendvps %xmm1, %xmm0, %xmm1, %xmm0
-; F16C-NEXT:    vzeroupper
 ; F16C-NEXT:    retq
 ;
 ; AVX512F-LABEL: fptoui_4f16_to_4i32:
@@ -5232,9 +5230,8 @@ define <4 x i32> @fptoui_4f16_to_4i32(<4 x half> %a) nounwind {
 ;
 ; AVX512-FASTLANE-LABEL: fptoui_4f16_to_4i32:
 ; AVX512-FASTLANE:       # %bb.0:
-; AVX512-FASTLANE-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-FASTLANE-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-FASTLANE-NEXT:    vcvttps2udq %xmm0, %xmm0
-; AVX512-FASTLANE-NEXT:    vzeroupper
 ; AVX512-FASTLANE-NEXT:    retq
   %cvt = fptoui <4 x half> %a to <4 x i32>
   ret <4 x i32> %cvt
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
index ccaaca6c44f050d..24c05ba9c52f6f1 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -2771,3 +2771,37 @@ entry:
   %vzip.i = shufflevector <16 x i8> %lane, <16 x i8> %lane3, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   ret <16 x i8> %vzip.i
 }
+
+define <8 x i16> @PR104482(<16 x i8> %i) {
+; SSE2-LABEL: PR104482:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: PR104482:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[15],zero,xmm0[14],zero,xmm0[13],zero,xmm0[12],zero,xmm0[11],zero,xmm0[10],zero,xmm0[9],zero,xmm0[8],zero
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: PR104482:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[15],zero,xmm0[14],zero,xmm0[13],zero,xmm0[12],zero,xmm0[11],zero,xmm0[10],zero,xmm0[9],zero,xmm0[8],zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: PR104482:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[15],zero,xmm0[14],zero,xmm0[13],zero,xmm0[12],zero,xmm0[11],zero,xmm0[10],zero,xmm0[9],zero,xmm0[8],zero
+; AVX-NEXT:    retq
+  %i7 = shufflevector <16 x i8> %i, <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  %1 = bitcast <16 x i8> %i7 to <8 x i16>
+  %i10 = shufflevector <8 x i16> %1, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %i11 = shufflevector <8 x i16> %i10, <8 x i16> poison, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
+  %i12 = shufflevector <8 x i16> %i11, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4>
+  ret <8 x i16> %i12
+}
diff --git a/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll b/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
index 9555ce032db90cb..732fc6543e31410 100644
--- a/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
+++ b/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
@@ -10,5 +10,5 @@ start:
 attributes #0 = { nonlazybind uwtable "probe-stack"="probe_stack" "target-cpu"="x86-64" }
 
 ; CHECK-LABEL: foo:
-; CHECK: movabsq $4294967304, %rax
+; CHECK: movabsq $4294967312, %rax
 ; CHECK-NEXT: callq probe_stack
diff --git a/llvm/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll b/llvm/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll
index bcfba68baa88f79..ca61770b5eddbd7 100644
--- a/llvm/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll
+++ b/llvm/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=asm -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -filetype=asm -asm-verbose=0 -arm-atomic-cfg-tidy=0 < %s | FileCheck %s
 
 ; int main()
 ; {
diff --git a/llvm/test/DebugInfo/COFF/AArch64/arm64-register-variables.ll b/llvm/test/DebugInfo/COFF/AArch64/arm64-register-variables.ll
index 834e8f4ce7fed16..950594ed105d631 100644
--- a/llvm/test/DebugInfo/COFF/AArch64/arm64-register-variables.ll
+++ b/llvm/test/DebugInfo/COFF/AArch64/arm64-register-variables.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=arm64-windows -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
+; RUN: llc < %s -mtriple=arm64-windows -filetype=obj -aarch64-enable-atomic-cfg-tidy=0 | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; Generated from:
 ; volatile int x;
@@ -41,56 +41,50 @@ target triple = "arm64-unknown-windows-msvc19.16.27023"
 
 @x = common dso_local global i32 0, align 4, !dbg !0
 
-; Function Attrs: noinline nounwind optnone uwtable
-define dso_local void @f(i32 %p) #0 !dbg !12 {
+define dso_local void @f(i32 %p) !dbg !11 {
 entry:
   %p.addr = alloca i32, align 4
   %a = alloca i32, align 4
   %b = alloca i32, align 4
   %c = alloca i32, align 4
   store i32 %p, ptr %p.addr, align 4
-  call void @llvm.dbg.declare(metadata ptr %p.addr, metadata !15, metadata !DIExpression()), !dbg !16
-  %0 = load i32, ptr %p.addr, align 4, !dbg !17
-  %tobool = icmp ne i32 %0, 0, !dbg !17
-  br i1 %tobool, label %if.then, label %if.else, !dbg !17
+    #dbg_declare(ptr %p.addr, !14, !DIExpression(), !15)
+  %0 = load i32, ptr %p.addr, align 4, !dbg !16
+  %tobool = icmp ne i32 %0, 0, !dbg !16
+  %call = call i32 @getint(), !dbg !17
+  br i1 %tobool, label %if.then, label %if.else, !dbg !16
 
 if.then:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata ptr %a, metadata !18, metadata !DIExpression()), !dbg !21
-  %call = call i32 @getint(), !dbg !21
+    #dbg_declare(ptr %a, !19, !DIExpression(), !21)
   store i32 %call, ptr %a, align 4, !dbg !21
-  call void @llvm.dbg.declare(metadata ptr %b, metadata !22, metadata !DIExpression()), !dbg !23
+    #dbg_declare(ptr %b, !22, !DIExpression(), !23)
   %1 = load i32, ptr %a, align 4, !dbg !23
   %call1 = call i32 @inlineinc(i32 %1), !dbg !23
   store i32 %call1, ptr %b, align 4, !dbg !23
   %2 = load i32, ptr %b, align 4, !dbg !24
-  call void @putint(i32 %2), !dbg !24
   br label %if.end, !dbg !25
 
 if.else:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata ptr %c, metadata !26, metadata !DIExpression()), !dbg !28
-  %call2 = call i32 @getint(), !dbg !28
-  store i32 %call2, ptr %c, align 4, !dbg !28
+    #dbg_declare(ptr %c, !26, !DIExpression(), !28)
+  store i32 %call, ptr %c, align 4, !dbg !28
   %3 = load i32, ptr %c, align 4, !dbg !29
-  call void @putint(i32 %3), !dbg !29
   br label %if.end, !dbg !30
 
 if.end:                                           ; preds = %if.else, %if.then
+  %.sink = phi i32 [ %3, %if.else ], [ %2, %if.then ]
+  call void @putint(i32 %.sink), !dbg !17
   ret void, !dbg !31
 }
 
-; Function Attrs: nounwind readnone speculatable
-declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+declare dso_local i32 @getint()
 
-declare dso_local i32 @getint() #2
-
-; Function Attrs: noinline nounwind optnone uwtable
-define internal i32 @inlineinc(i32 %a) #0 !dbg !32 {
+define internal i32 @inlineinc(i32 %a) !dbg !32 {
 entry:
   %a.addr = alloca i32, align 4
   %b = alloca i32, align 4
   store i32 %a, ptr %a.addr, align 4
-  call void @llvm.dbg.declare(metadata ptr %a.addr, metadata !35, metadata !DIExpression()), !dbg !36
-  call void @llvm.dbg.declare(metadata ptr %b, metadata !37, metadata !DIExpression()), !dbg !38
+    #dbg_declare(ptr %a.addr, !35, !DIExpression(), !36)
+    #dbg_declare(ptr %b, !37, !DIExpression(), !38)
   %0 = load i32, ptr %a.addr, align 4, !dbg !38
   %add = add nsw i32 %0, 1, !dbg !38
   store i32 %add, ptr %b, align 4, !dbg !38
@@ -101,7 +95,7 @@ entry:
   ret i32 %2, !dbg !40
 }
 
-declare dso_local void @putint(i32) #2
+declare dso_local void @putint(i32)
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!8, !9, !10}
@@ -109,7 +103,7 @@ declare dso_local void @putint(i32) #2
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "x", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
 !2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 9.0.0 (trunk 361867) (llvm/trunk 361866)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
-!3 = !DIFile(filename: "t.c", directory: "S:\5CLLVM\5Csvn\5Csbr\5Cbin", checksumkind: CSK_MD5, checksum: "734c448e95a6204a439a847ed063e5ce")
+!3 = !DIFile(filename: "t.c", directory: "S:\\LLVM\\svn\\sbr\\bin", checksumkind: CSK_MD5, checksum: "734c448e95a6204a439a847ed063e5ce")
 !4 = !{}
 !5 = !{!0}
 !6 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
@@ -117,27 +111,27 @@ declare dso_local void @putint(i32) #2
 !8 = !{i32 2, !"CodeView", i32 1}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{i32 1, !"wchar_size", i32 2}
-!11 = !{!"clang version 9.0.0 (trunk 361867) (llvm/trunk 361866)"}
-!12 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 9, type: !13, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
-!13 = !DISubroutineType(types: !14)
-!14 = !{null, !7}
-!15 = !DILocalVariable(name: "p", arg: 1, scope: !12, file: !3, line: 9, type: !7)
-!16 = !DILocation(line: 9, scope: !12)
-!17 = !DILocation(line: 10, scope: !12)
-!18 = !DILocalVariable(name: "a", scope: !19, file: !3, line: 11, type: !7)
-!19 = distinct !DILexicalBlock(scope: !20, file: !3, line: 10)
-!20 = distinct !DILexicalBlock(scope: !12, file: !3, line: 10)
-!21 = !DILocation(line: 11, scope: !19)
-!22 = !DILocalVariable(name: "b", scope: !19, file: !3, line: 12, type: !7)
-!23 = !DILocation(line: 12, scope: !19)
-!24 = !DILocation(line: 13, scope: !19)
-!25 = !DILocation(line: 14, scope: !19)
+!11 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 9, type: !12, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !7}
+!14 = !DILocalVariable(name: "p", arg: 1, scope: !11, file: !3, line: 9, type: !7)
+!15 = !DILocation(line: 9, scope: !11)
+!16 = !DILocation(line: 10, scope: !11)
+!17 = !DILocation(line: 0, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !11, file: !3, line: 10)
+!19 = !DILocalVariable(name: "a", scope: !20, file: !3, line: 11, type: !7)
+!20 = distinct !DILexicalBlock(scope: !18, file: !3, line: 10)
+!21 = !DILocation(line: 11, scope: !20)
+!22 = !DILocalVariable(name: "b", scope: !20, file: !3, line: 12, type: !7)
+!23 = !DILocation(line: 12, scope: !20)
+!24 = !DILocation(line: 13, scope: !20)
+!25 = !DILocation(line: 14, scope: !20)
 !26 = !DILocalVariable(name: "c", scope: !27, file: !3, line: 15, type: !7)
-!27 = distinct !DILexicalBlock(scope: !20, file: !3, line: 14)
+!27 = distinct !DILexicalBlock(scope: !18, file: !3, line: 14)
 !28 = !DILocation(line: 15, scope: !27)
 !29 = !DILocation(line: 16, scope: !27)
 !30 = !DILocation(line: 17, scope: !27)
-!31 = !DILocation(line: 18, scope: !12)
+!31 = !DILocation(line: 18, scope: !11)
 !32 = distinct !DISubprogram(name: "inlineinc", scope: !3, file: !3, line: 4, type: !33, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, retainedNodes: !4)
 !33 = !DISubroutineType(types: !34)
 !34 = !{!7, !7}
diff --git a/llvm/test/DebugInfo/COFF/ARMNT/arm-register-variables.ll b/llvm/test/DebugInfo/COFF/ARMNT/arm-register-variables.ll
index 1eb8dff00e58938..079c14eeb3abf3e 100644
--- a/llvm/test/DebugInfo/COFF/ARMNT/arm-register-variables.ll
+++ b/llvm/test/DebugInfo/COFF/ARMNT/arm-register-variables.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
+; RUN: llc < %s -filetype=obj -arm-atomic-cfg-tidy=0 | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; Generated from:
 
@@ -57,7 +57,6 @@ target triple = "thumbv7-pc-windows-msvc19.11.0"
 
 @x = dso_local global i32 0, align 4, !dbg !0
 
-; Function Attrs: noinline nounwind optnone
 define dso_local arm_aapcs_vfpcc void @f(i32 %p) !dbg !14 {
 entry:
   %p.addr = alloca i32, align 4
@@ -65,56 +64,51 @@ entry:
   %b = alloca i32, align 4
   %c = alloca i32, align 4
   store i32 %p, ptr %p.addr, align 4
-  call void @llvm.dbg.declare(metadata ptr %p.addr, metadata !17, metadata !DIExpression()), !dbg !18
+    #dbg_declare(ptr %p.addr, !17, !DIExpression(), !18)
   %0 = load i32, ptr %p.addr, align 4, !dbg !19
   %tobool = icmp ne i32 %0, 0, !dbg !19
+  %call = call arm_aapcs_vfpcc i32 @getint(), !dbg !20
   br i1 %tobool, label %if.then, label %if.else, !dbg !19
 
 if.then:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata ptr %a, metadata !20, metadata !DIExpression()), !dbg !23
-  %call = call arm_aapcs_vfpcc i32 @getint(), !dbg !23
-  store i32 %call, ptr %a, align 4, !dbg !23
-  call void @llvm.dbg.declare(metadata ptr %b, metadata !24, metadata !DIExpression()), !dbg !25
-  %1 = load i32, ptr %a, align 4, !dbg !25
-  %call1 = call arm_aapcs_vfpcc i32 @inlineinc(i32 %1), !dbg !25
-  store i32 %call1, ptr %b, align 4, !dbg !25
-  %2 = load i32, ptr %b, align 4, !dbg !26
-  call arm_aapcs_vfpcc void @putint(i32 %2), !dbg !26
-  br label %if.end, !dbg !27
+    #dbg_declare(ptr %a, !22, !DIExpression(), !24)
+  store i32 %call, ptr %a, align 4, !dbg !24
+    #dbg_declare(ptr %b, !25, !DIExpression(), !26)
+  %1 = load i32, ptr %a, align 4, !dbg !26
+  %call1 = call arm_aapcs_vfpcc i32 @inlineinc(i32 %1), !dbg !26
+  store i32 %call1, ptr %b, align 4, !dbg !26
+  %2 = load i32, ptr %b, align 4, !dbg !27
+  br label %if.end, !dbg !28
 
 if.else:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata ptr %c, metadata !28, metadata !DIExpression()), !dbg !30
-  %call2 = call arm_aapcs_vfpcc i32 @getint(), !dbg !30
-  store i32 %call2, ptr %c, align 4, !dbg !30
-  %3 = load i32, ptr %c, align 4, !dbg !31
-  call arm_aapcs_vfpcc void @putint(i32 %3), !dbg !31
-  br label %if.end, !dbg !32
+    #dbg_declare(ptr %c, !29, !DIExpression(), !31)
+  store i32 %call, ptr %c, align 4, !dbg !31
+  %3 = load i32, ptr %c, align 4, !dbg !32
+  br label %if.end, !dbg !33
 
 if.end:                                           ; preds = %if.else, %if.then
-  ret void, !dbg !33
+  %.sink = phi i32 [ %3, %if.else ], [ %2, %if.then ]
+  call arm_aapcs_vfpcc void @putint(i32 %.sink), !dbg !20
+  ret void, !dbg !34
 }
 
-; Function Attrs: nounwind readnone speculatable willreturn
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
 declare dso_local arm_aapcs_vfpcc i32 @getint()
 
-; Function Attrs: noinline nounwind optnone
-define internal arm_aapcs_vfpcc i32 @inlineinc(i32 %a) !dbg !34 {
+define internal arm_aapcs_vfpcc i32 @inlineinc(i32 %a) !dbg !35 {
 entry:
   %a.addr = alloca i32, align 4
   %b = alloca i32, align 4
   store i32 %a, ptr %a.addr, align 4
-  call void @llvm.dbg.declare(metadata ptr %a.addr, metadata !37, metadata !DIExpression()), !dbg !38
-  call void @llvm.dbg.declare(metadata ptr %b, metadata !39, metadata !DIExpression()), !dbg !40
-  %0 = load i32, ptr %a.addr, align 4, !dbg !40
-  %add = add nsw i32 %0, 1, !dbg !40
-  store i32 %add, ptr %b, align 4, !dbg !40
-  %1 = load volatile i32, ptr @x, align 4, !dbg !41
-  %inc = add nsw i32 %1, 1, !dbg !41
-  store volatile i32 %inc, ptr @x, align 4, !dbg !41
-  %2 = load i32, ptr %b, align 4, !dbg !42
-  ret i32 %2, !dbg !42
+    #dbg_declare(ptr %a.addr, !38, !DIExpression(), !39)
+    #dbg_declare(ptr %b, !40, !DIExpression(), !41)
+  %0 = load i32, ptr %a.addr, align 4, !dbg !41
+  %add = add nsw i32 %0, 1, !dbg !41
+  store i32 %add, ptr %b, align 4, !dbg !41
+  %1 = load volatile i32, ptr @x, align 4, !dbg !42
+  %inc = add nsw i32 %1, 1, !dbg !42
+  store volatile i32 %inc, ptr @x, align 4, !dbg !42
+  %2 = load i32, ptr %b, align 4, !dbg !43
+  ret i32 %2, !dbg !43
 }
 
 declare dso_local arm_aapcs_vfpcc void @putint(i32)
@@ -143,26 +137,27 @@ declare dso_local arm_aapcs_vfpcc void @putint(i32)
 !17 = !DILocalVariable(name: "p", arg: 1, scope: !14, file: !6, line: 9, type: !8)
 !18 = !DILocation(line: 9, scope: !14)
 !19 = !DILocation(line: 10, scope: !14)
-!20 = !DILocalVariable(name: "a", scope: !21, file: !6, line: 11, type: !8)
-!21 = distinct !DILexicalBlock(scope: !22, file: !6, line: 10)
-!22 = distinct !DILexicalBlock(scope: !14, file: !6, line: 10)
-!23 = !DILocation(line: 11, scope: !21)
-!24 = !DILocalVariable(name: "b", scope: !21, file: !6, line: 12, type: !8)
-!25 = !DILocation(line: 12, scope: !21)
-!26 = !DILocation(line: 13, scope: !21)
-!27 = !DILocation(line: 14, scope: !21)
-!28 = !DILocalVariable(name: "c", scope: !29, file: !6, line: 15, type: !8)
-!29 = distinct !DILexicalBlock(scope: !22, file: !6, line: 14)
-!30 = !DILocation(line: 15, scope: !29)
-!31 = !DILocation(line: 16, scope: !29)
-!32 = !DILocation(line: 17, scope: !29)
-!33 = !DILocation(line: 18, scope: !14)
-!34 = distinct !DISubprogram(name: "inlineinc", scope: !6, file: !6, line: 4, type: !35, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, retainedNodes: !4)
-!35 = !DISubroutineType(types: !36)
-!36 = !{!8, !8}
-!37 = !DILocalVariable(name: "a", arg: 1, scope: !34, file: !6, line: 4, type: !8)
-!38 = !DILocation(line: 4, scope: !34)
-!39 = !DILocalVariable(name: "b", scope: !34, file: !6, line: 5, type: !8)
-!40 = !DILocation(line: 5, scope: !34)
-!41 = !DILocation(line: 6, scope: !34)
-!42 = !DILocation(line: 7, scope: !34)
+!20 = !DILocation(line: 0, scope: !21)
+!21 = distinct !DILexicalBlock(scope: !14, file: !6, line: 10)
+!22 = !DILocalVariable(name: "a", scope: !23, file: !6, line: 11, type: !8)
+!23 = distinct !DILexicalBlock(scope: !21, file: !6, line: 10)
+!24 = !DILocation(line: 11, scope: !23)
+!25 = !DILocalVariable(name: "b", scope: !23, file: !6, line: 12, type: !8)
+!26 = !DILocation(line: 12, scope: !23)
+!27 = !DILocation(line: 13, scope: !23)
+!28 = !DILocation(line: 14, scope: !23)
+!29 = !DILocalVariable(name: "c", scope: !30, file: !6, line: 15, type: !8)
+!30 = distinct !DILexicalBlock(scope: !21, file: !6, line: 14)
+!31 = !DILocation(line: 15, scope: !30)
+!32 = !DILocation(line: 16, scope: !30)
+!33 = !DILocation(line: 17, scope: !30)
+!34 = !DILocation(line: 18, scope: !14)
+!35 = distinct !DISubprogram(name: "inlineinc", scope: !6, file: !6, line: 4, type: !36, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!36 = !DISubroutineType(types: !37)
+!37 = !{!8, !8}
+!38 = !DILocalVariable(name: "a", arg: 1, scope: !35, file: !6, line: 4, type: !8)
+!39 = !DILocation(line: 4, scope: !35)
+!40 = !DILocalVariable(name: "b", scope: !35, file: !6, line: 5, type: !8)
+!41 = !DILocation(line: 5, scope: !35)
+!42 = !DILocation(line: 6, scope: !35)
+!43 = !DILocation(line: 7, scope: !35)
diff --git a/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll b/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
index 5432b59d583bacf..1a467ec72a75da2 100644
--- a/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
+++ b/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
@@ -5,7 +5,7 @@
 ;; AIX assembly syntax.
 
 ; REQUIRES: powerpc-registered-target
-; RUN: llc -filetype=obj -o %t -mtriple=powerpc-aix-ibm-xcoff < %s
+; RUN: llc -filetype=obj -o %t -mtriple=powerpc-aix-ibm-xcoff -ppc-global-merge=false < %s
 ; RUN: llvm-symbolizer --obj=%t 'DATA 0x60' 'DATA 0x61' 'DATA 0x64' 'DATA 0X68' \
 ; RUN:   'DATA 0x90' 'DATA 0x94' 'DATA 0X98' | \
 ; RUN:   FileCheck %s
diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test b/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test
new file mode 100644
index 000000000000000..f537aaee2ef4835
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test
@@ -0,0 +1,11 @@
+# RUN: llc -filetype=obj -mtriple x86_64-apple-macosx -o %t.x86-64.o \
+# RUN:   %S/Inputs/main-ret-0.ll
+# RUN: llc -filetype=obj -mtriple arm64-apple-macosx -o %t.arm64.o \
+# RUN:   %S/Inputs/main-ret-0.ll
+# RUN: llvm-lipo -create -output %t.universal.o -arch x86_64 %t.x86-64.o \
+# RUN:   -arch arm64 %t.arm64.o
+# RUN: llvm-jitlink -noexec -triple arm64-apple-macosx %t.universal.o
+#
+# REQUIRES: x86-registered-target && aarch64-registered-target
+#
+# Check that universal binaries don't raise "not a valid object file" errors.
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
index 62c0a24c60d77f1..e986f1085426887 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
@@ -62,7 +62,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    store i64 1102416563, ptr [[TMP23]], align 8
 ; CHECK-INLINE-NEXT:    [[TMP24:%.*]] = add i64 [[TMP12]], 8
 ; CHECK-INLINE-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
-; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP25]], align 8
+; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP25]], align 8
 ; CHECK-INLINE-NEXT:    [[TMP26:%.*]] = add i64 [[TMP12]], 16
 ; CHECK-INLINE-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr
 ; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP27]], align 8
@@ -95,7 +95,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP42:%.*]] = inttoptr i64 [[TMP41]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP44:%.*]] = icmp ne i8 [[TMP43]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP44]], label [[TMP45:%.*]], label [[TMP50:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP44]], label [[TMP45:%.*]], label [[TMP50:%.*]], !prof [[PROF1:![0-9]+]]
 ; CHECK-INLINE:       45:
 ; CHECK-INLINE-NEXT:    [[TMP46:%.*]] = and i64 [[TMP21]], 7
 ; CHECK-INLINE-NEXT:    [[TMP47:%.*]] = trunc i64 [[TMP46]] to i8
@@ -153,7 +153,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP82:%.*]] = inttoptr i64 [[TMP81]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP83:%.*]] = load i8, ptr [[TMP82]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP84:%.*]] = icmp ne i8 [[TMP83]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP84]], label [[TMP85:%.*]], label [[TMP90:%.*]], !prof [[PROF0]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP84]], label [[TMP85:%.*]], label [[TMP90:%.*]], !prof [[PROF1]]
 ; CHECK-INLINE:       85:
 ; CHECK-INLINE-NEXT:    [[TMP86:%.*]] = and i64 [[TMP77]], 7
 ; CHECK-INLINE-NEXT:    [[TMP87:%.*]] = trunc i64 [[TMP86]] to i8
@@ -193,7 +193,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP101:%.*]] = inttoptr i64 [[TMP100]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP102:%.*]] = load i8, ptr [[TMP101]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP103:%.*]] = icmp ne i8 [[TMP102]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP103]], label [[TMP104:%.*]], label [[TMP109:%.*]], !prof [[PROF0]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP103]], label [[TMP104:%.*]], label [[TMP109:%.*]], !prof [[PROF1]]
 ; CHECK-INLINE:       104:
 ; CHECK-INLINE-NEXT:    [[TMP105:%.*]] = and i64 [[TMP54]], 7
 ; CHECK-INLINE-NEXT:    [[TMP106:%.*]] = trunc i64 [[TMP105]] to i8
@@ -221,7 +221,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP119:%.*]] = inttoptr i64 [[TMP118]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP120:%.*]] = load i8, ptr [[TMP119]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP121:%.*]] = icmp ne i8 [[TMP120]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP121]], label [[TMP122:%.*]], label [[TMP127:%.*]], !prof [[PROF0]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP121]], label [[TMP122:%.*]], label [[TMP127:%.*]], !prof [[PROF1]]
 ; CHECK-INLINE:       122:
 ; CHECK-INLINE-NEXT:    [[TMP123:%.*]] = and i64 [[TMP116]], 7
 ; CHECK-INLINE-NEXT:    [[TMP124:%.*]] = trunc i64 [[TMP123]] to i8
@@ -236,7 +236,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP130:%.*]] = inttoptr i64 [[TMP129]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP131:%.*]] = load i8, ptr [[TMP130]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP132:%.*]] = icmp ne i8 [[TMP131]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP132]], label [[TMP133:%.*]], label [[EHEXIT:%.*]], !prof [[PROF0]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP132]], label [[TMP133:%.*]], label [[EHEXIT:%.*]], !prof [[PROF1]]
 ; CHECK-INLINE:       133:
 ; CHECK-INLINE-NEXT:    [[TMP134:%.*]] = and i64 [[TMP114]], 7
 ; CHECK-INLINE-NEXT:    [[TMP135:%.*]] = trunc i64 [[TMP134]] to i8
@@ -308,7 +308,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-OUTLINE-NEXT:    store i64 1102416563, ptr [[TMP27]], align 8
 ; CHECK-OUTLINE-NEXT:    [[TMP28:%.*]] = add i64 [[TMP12]], 8
 ; CHECK-OUTLINE-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
-; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP29]], align 8
+; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP29]], align 8
 ; CHECK-OUTLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP12]], 16
 ; CHECK-OUTLINE-NEXT:    [[TMP31:%.*]] = inttoptr i64 [[TMP30]] to ptr
 ; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP31]], align 8
@@ -524,7 +524,7 @@ define void @OtherPersonality(ptr %ptrParam) sanitize_address personality ptr @d
 ; CHECK-NEXT:    store i64 1102416563, ptr [[TMP14]], align 8
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP11]], 8
 ; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
-; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_.1 to i64), ptr [[TMP16]], align 8
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP16]], align 8
 ; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP11]], 16
 ; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
 ; CHECK-NEXT:    store i64 ptrtoint (ptr @OtherPersonality to i64), ptr [[TMP18]], align 8
@@ -580,5 +580,5 @@ ehcleanup:                                        ; preds = %entry
   cleanupret from %0 unwind to caller
 }
 ;.
-; CHECK-INLINE: [[PROF0]] = !{!"branch_weights", i32 1, i32 1048575}
+; CHECK-INLINE: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
 ;.
diff --git a/llvm/test/Instrumentation/AddressSanitizer/fake-stack.ll b/llvm/test/Instrumentation/AddressSanitizer/fake-stack.ll
index ea2a21ec3b8db0c..3cccabb13cbc6f6 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/fake-stack.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/fake-stack.ll
@@ -27,7 +27,7 @@ define void @Simple() uwtable sanitize_address {
 ; NEVER-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
 ; NEVER-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
 ; NEVER-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
-; NEVER-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP5]], align 8
+; NEVER-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP5]], align 8
 ; NEVER-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
 ; NEVER-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; NEVER-NEXT:    store i64 ptrtoint (ptr @Simple to i64), ptr [[TMP7]], align 8
@@ -69,7 +69,7 @@ define void @Simple() uwtable sanitize_address {
 ; RUNTIME-NEXT:    store i64 1102416563, ptr [[TMP13]], align 8
 ; RUNTIME-NEXT:    [[TMP14:%.*]] = add i64 [[TMP10]], 8
 ; RUNTIME-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
-; RUNTIME-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP15]], align 8
+; RUNTIME-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP15]], align 8
 ; RUNTIME-NEXT:    [[TMP16:%.*]] = add i64 [[TMP10]], 16
 ; RUNTIME-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
 ; RUNTIME-NEXT:    store i64 ptrtoint (ptr @Simple to i64), ptr [[TMP17]], align 8
@@ -119,7 +119,7 @@ define void @Simple() uwtable sanitize_address {
 ; ALWAYS-NEXT:    store i64 1102416563, ptr [[TMP8]], align 8
 ; ALWAYS-NEXT:    [[TMP9:%.*]] = add i64 [[TMP5]], 8
 ; ALWAYS-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
-; ALWAYS-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP10]], align 8
+; ALWAYS-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP10]], align 8
 ; ALWAYS-NEXT:    [[TMP11:%.*]] = add i64 [[TMP5]], 16
 ; ALWAYS-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; ALWAYS-NEXT:    store i64 ptrtoint (ptr @Simple to i64), ptr [[TMP12]], align 8
@@ -167,7 +167,7 @@ define void @Huge() uwtable sanitize_address {
 ; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
-; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_.1 to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    store i64 ptrtoint (ptr @Huge to i64), ptr [[TMP7]], align 8
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
index 47df1d43549cec4..6883a99be174952 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
@@ -20,7 +20,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: @.str = internal constant { [14 x i8], [18 x i8] } { [14 x i8] c"Hello, world!\00", [18 x i8] zeroinitializer }, comdat({{.*}}), align 32
 
 ; Check emitted location descriptions:
-; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
+; CHECK: [[VARNAME:@___asan_gen_global[.0-9]*]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
 ; NOALIAS: @__asan_global_global = {{.*}}i64 ptrtoint (ptr @global to i64){{.*}} section "asan_globals"{{.*}}, !associated
 ; NOALIAS: @__asan_global_.str = {{.*}}i64 ptrtoint (ptr @{{.str|1}} to i64){{.*}} section "asan_globals"{{.*}}, !associated
 ; ALIAS: @__asan_global_global = {{.*}}i64 ptrtoint (ptr @0 to i64){{.*}} section "asan_globals"{{.*}}, !associated
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
index 1617bf9b67aa7c5..ab1d5ffc2ef3434 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
@@ -1,10 +1,10 @@
-; RUN: rm -rf %t && split-file %s %t && cd %t
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-apple-macosx10.11.0 -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-pc-windows-msvc19.0.24215 -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -asan-mapping-scale=5 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefixes=CHECK,CHECK-S5 %s
+; RUN: rm -rf %t && split-file %s %t
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-apple-macosx10.11.0 -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-pc-windows-msvc19.0.24215 -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=0 -asan-mapping-scale=5 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefixes=CHECK,CHECK-S5 %s
 
-; RUN: opt < empty.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=ELF-NOGC %s
+; RUN: opt < %t/empty.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=ELF-NOGC %s
 
 ;--- a.ll
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -22,7 +22,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; CHECK: @.str = internal constant { [14 x i8], [18 x i8] } { [14 x i8] c"Hello, world!\00", [18 x i8] zeroinitializer }{{.*}}, align 32
 
 ; Check emitted location descriptions:
-; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
+; CHECK: [[VARNAME:@___asan_gen_global[.0-9]*]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
 
 ; Check that location descriptors and global names were passed into __asan_register_globals:
 ; CHECK: i64 ptrtoint (ptr [[VARNAME]] to i64)
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
index f04eeb4b7aa4031..b2578292406bc58 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
@@ -4,12 +4,12 @@
 ; We keep using comdats for garbage collection if odr indicators are
 ; enabled as indicator symbols will cause link time odr violations.
 ; This is to fix PR 47925.
-; RUN: rm -rf %t && split-file %s %t && cd %t
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=1 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK,NOCOMDAT
+; RUN: rm -rf %t && split-file %s %t
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=1 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK,NOCOMDAT
 ; Check that enabling odr indicators enables comdat for globals.
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,COMDAT
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,COMDAT
 
-; RUN: opt < no_module_id.ll -passes=asan -S | FileCheck %s --check-prefix=NOMODULEID
+; RUN: opt < %t/no_module_id.ll -passes=asan -S | FileCheck %s --check-prefix=NOMODULEID
 
 ;--- a.ll
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -46,9 +46,9 @@ target triple = "x86_64-unknown-linux-gnu"
 ; NOCOMDAT: @.str = internal constant { [14 x i8], [18 x i8] } { [14 x i8] c"Hello, world!\00", [18 x i8] zeroinitializer }, align 32
 
 ; Check emitted location descriptions:
-; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
+; CHECK: [[VARNAME:@___asan_gen_global[.0-9]*]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
 ; COMDAT: @__asan_global_global = {{.*}}i64 ptrtoint (ptr @__odr_asan_gen_global to i64){{.*}} section "asan_globals"{{.*}}, comdat($global), !associated
-; COMDAT: @__asan_global_.str = {{.*}}i64 ptrtoint (ptr @___asan_gen_ to i64){{.*}} section "asan_globals"{{.*}}, comdat($.str.{{.*}}), !associated
+; COMDAT: @__asan_global_.str = {{.*}}i64 ptrtoint (ptr @___asan_gen_global{{[.0-9]*}} to i64){{.*}} section "asan_globals"{{.*}}, comdat($.str.{{.*}}), !associated
 
 ; The metadata has to be inserted to llvm.compiler.used to avoid being stripped
 ; during LTO.
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_global.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_global.ll
index aa943cff6c4936e..2f26a7d9f44535e 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/instrument_global.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_global.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; If a global is present, __asan_[un]register_globals should be called from
 ; module ctor/dtor
 
-; CHECK: @___asan_gen_ = private constant [8 x i8] c"<stdin>\00", align 1
+; CHECK: @___asan_gen_module = private constant [8 x i8] c"<stdin>\00", align 1
 ; CHECK: @llvm.used = appending global [2 x ptr] [ptr @asan.module_ctor, ptr @asan.module_dtor], section "llvm.metadata"
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @asan.module_ctor, ptr @asan.module_ctor }]
 ; CHECK: @llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @asan.module_dtor, ptr @asan.module_dtor }]
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
new file mode 100644
index 000000000000000..b20de3eb80b786c
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --global-value-regex ".*global_ctors.*" --version 5
+; RUN: opt < %s -passes=asan -S | FileCheck %s
+; RUN: opt < %s -passes=asan -S -asan-initialization-order=0 | FileCheck %s --check-prefixes=NOINIT
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @__late_ctor, ptr null }]
+
+declare void @initializer() uwtable;
+
+;.
+; CHECK: @llvm.global_ctors = appending global [2 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @__late_ctor, ptr null }, { i32, ptr, ptr } { i32 1, ptr @asan.module_ctor, ptr @asan.module_ctor }]
+;.
+; NOINIT: @llvm.global_ctors = appending global [2 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @__late_ctor, ptr null }, { i32, ptr, ptr } { i32 1, ptr @asan.module_ctor, ptr @asan.module_ctor }]
+;.
+define internal void @__late_ctor() sanitize_address section ".text.startup" {
+; CHECK-LABEL: define internal void @__late_ctor(
+; CHECK-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @__asan_before_dynamic_init(i64 ptrtoint (ptr @___asan_gen_module to i64))
+; CHECK-NEXT:    call void @initializer()
+; CHECK-NEXT:    call void @__asan_after_dynamic_init()
+; CHECK-NEXT:    ret void
+;
+; NOINIT-LABEL: define internal void @__late_ctor(
+; NOINIT-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
+; NOINIT-NEXT:  [[ENTRY:.*:]]
+; NOINIT-NEXT:    call void @initializer()
+; NOINIT-NEXT:    ret void
+;
+entry:
+
+  call void @initializer()
+  ret void
+}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_late_initializer.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_late_initializer.ll
index 45d526a42c9f7b9..0c946f5934661f4 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/instrument_late_initializer.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_late_initializer.ll
@@ -51,7 +51,7 @@ define internal void @__late_ctor() sanitize_address section ".text.startup" {
 ; CHECK-LABEL: define internal void @__late_ctor(
 ; CHECK-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    call void @__asan_before_dynamic_init(i64 ptrtoint (ptr @___asan_gen_ to i64))
+; CHECK-NEXT:    call void @__asan_before_dynamic_init(i64 ptrtoint (ptr @___asan_gen_module to i64))
 ; CHECK-NEXT:    call void @__asan_after_dynamic_init()
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll b/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
index ada8120155e5efc..15496274445036f 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
@@ -18,9 +18,9 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Check that we generate internal alias and odr indicator symbols for global to be protected.
 ; CHECK-NOINDICATOR-NOT: __odr_asan_gen_a
 ; CHECK-NOALIAS-NOT: private alias
-; CHECK-INDICATOR: @___asan_gen_.1 = private unnamed_addr constant [2 x i8] c"a\00", align 1
+; CHECK-INDICATOR: @___asan_gen_global{{[.0-9]*}} = private unnamed_addr constant [2 x i8] c"a\00", align 1
 ; CHECK-INDICATOR: @__odr_asan_gen_a = global i8 0, align 1
-; CHECK-INDICATOR: @___asan_gen_.4 = private unnamed_addr constant [2 x i8] c"d\00", align 1
+; CHECK-INDICATOR: @___asan_gen_global{{[.0-9]*}} = private unnamed_addr constant [2 x i8] c"d\00", align 1
 ; CHECK-INDICATOR: @__odr_asan_gen__ZL1d = global i8 0, align 1
 ; CHECK-ALIAS: @0 = private alias { [2 x i32], [24 x i8] }, ptr @a
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll b/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
index 3160802b1e09e33..28e58a52ddb982c 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
@@ -9,15 +9,15 @@ target triple = "x86_64-unknown-linux-gnu"
 @c = internal global [2 x i32] zeroinitializer, align 4
 @d = unnamed_addr global [2 x i32] zeroinitializer, align 4
 
-; NOALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @a to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.1 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
-; NOALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @b to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.2 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-; NOALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @c to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.3 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-; NOALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @d to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.4 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; NOALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @a to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }
+; NOALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @b to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 -1 }
+; NOALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @c to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 -1 }
+; NOALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @d to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }
 
-; ALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @0 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.1 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
-; ALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @1 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.2 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-; ALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @2 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.3 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-; ALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @3 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.4 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; ALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @0 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }
+; ALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @1 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 -1 }
+; ALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @2 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 -1 }
+; ALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @3 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }
 ; ALIAS:      @0 = private alias {{.*}} @a
 ; ALIAS-NEXT: @1 = private alias {{.*}} @b
 ; ALIAS-NEXT: @2 = private alias {{.*}} @c
diff --git a/llvm/test/Instrumentation/AddressSanitizer/win-string-literal.ll b/llvm/test/Instrumentation/AddressSanitizer/win-string-literal.ll
index ac4bc4623e1ebf2..41691f0fc8dd4e5 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/win-string-literal.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/win-string-literal.ll
@@ -13,7 +13,7 @@
 ; CHECK:      @"__asan_global_??_C@_04JIHMPGLA@asdf?$AA@" =
 ; CHECK-SAME: private global { i64, i64, i64, i64, i64, i64, i64, i64 }
 ; CHECK-SAME: { i64 ptrtoint (ptr @"??_C@_04JIHMPGLA@asdf?$AA@" to i64),
-; CHECK-SAME:   i64 5, i64 32, i64 ptrtoint (ptr @___asan_gen_.1 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }, section ".ASAN$GL",
+; CHECK-SAME:   i64 5, i64 32, i64 ptrtoint (ptr @___asan_gen_global to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }, section ".ASAN$GL",
 ; CHECK-SAME:   comdat($"??_C@_04JIHMPGLA@asdf?$AA@"), align 64
 
 ; ModuleID = 't.cpp'
diff --git a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
new file mode 100644
index 000000000000000..998443ec0cd568e
--- /dev/null
+++ b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -S -passes=instrprof -conditional-counter-update | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@__profn_foo = private constant [3 x i8] c"foo"
+@__profn_bar = private constant [3 x i8] c"bar"
+
+; CHECK-LABEL: define void @foo
+; CHECK-NEXT: %pgocount = load i8, ptr @__profc_foo, align 1
+; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero, label %1, label %2
+
+; CHECK-LABEL: 1:
+; CHECK-NEXT: store i8 0, ptr @__profc_foo, align 1
+; CHECK-NEXT: br label %2
+
+; CHECK-LABEL: 2:
+; CHECK-NEXT: ret void
+define void @foo() {
+  call void @llvm.instrprof.cover(ptr @__profn_foo, i64 0, i32 1, i32 0)
+  ret void
+}
+
+; CHECK-LABEL: define i32 @bar
+; CHECK-LABEL: entry:
+; CHECK-NEXT: %retval = alloca i32, align 4
+; CHECK-NEXT: %cond.addr = alloca i32, align 4
+; CHECK-NEXT: store i32 %cond, ptr %cond.addr, align 4
+; CHECK-NEXT: %pgocount = load i8, ptr @__profc_bar, align 1
+; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero, label %0, label %1
+
+; CHECK-LABEL: 0:                                  ; preds = %entry
+; CHECK-NEXT: store i8 0, ptr @__profc_bar, align 1
+; CHECK-NEXT: br label %1
+
+; CHECK-LABEL: 1:                                  ; preds = %entry, %0
+; CHECK-NEXT: %2 = load i32, ptr %cond.addr, align 4
+; CHECK-NEXT: %cmp = icmp slt i32 %2, 0
+; CHECK-NEXT: br i1 %cmp, label %if.then, label %if.end
+
+; CHECK-LABEL: if.then:                            ; preds = %1
+; CHECK-NEXT: %pgocount1 = load i8, ptr getelementptr inbounds ([3 x i8], ptr @__profc_bar, i32 0, i32 1), align 1
+; CHECK-NEXT: %pgocount.ifnonzero2 = icmp ne i8 %pgocount1, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero2, label %3, label %4
+define i32 @bar(i32 %cond) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %cond.addr = alloca i32, align 4
+  store i32 %cond, ptr %cond.addr, align 4
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 0)
+  %0 = load i32, ptr %cond.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 1)
+  store i32 -1, ptr %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 2)
+  store i32 0, ptr %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %1 = load i32, ptr %retval, align 4
+  ret i32 %1
+}
diff --git a/llvm/test/MC/AArch64/arm64-system-encoding.s b/llvm/test/MC/AArch64/arm64-system-encoding.s
index 313ec9117746035..c58a8f0cb841cb8 100644
--- a/llvm/test/MC/AArch64/arm64-system-encoding.s
+++ b/llvm/test/MC/AArch64/arm64-system-encoding.s
@@ -1,5 +1,5 @@
 ; RUN: not llvm-mc -triple arm64-apple-darwin -show-encoding < %s 2> %t | FileCheck %s
-; RUN: not llvm-mc -triple arm64-apple-darwin -mattr=+v8.3a -show-encoding < %s 2> %t | FileCheck %s --check-prefix=CHECK-V83
+; RUN: not llvm-mc -triple arm64-apple-darwin -mattr=+ccidx -show-encoding < %s 2> %t | FileCheck %s --check-prefix=CHECK-V83
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 foo:
diff --git a/llvm/test/MC/AArch64/arm64-target-specific-sysreg.s b/llvm/test/MC/AArch64/arm64-target-specific-sysreg.s
deleted file mode 100644
index 05cea3ac2da55f4..000000000000000
--- a/llvm/test/MC/AArch64/arm64-target-specific-sysreg.s
+++ /dev/null
@@ -1,10 +0,0 @@
-// RUN: not llvm-mc -triple arm64 -mcpu=generic -show-encoding < %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=CHECK-GENERIC
-//
-// RUN: llvm-mc -triple arm64 -mcpu=cyclone -show-encoding < %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=CHECK-CYCLONE
-
-msr CPM_IOACC_CTL_EL3, x0
-
-// CHECK-GENERIC: error: expected writable system register or pstate
-// CHECK-CYCLONE: msr CPM_IOACC_CTL_EL3, x0   // encoding: [0x00,0xf2,0x1f,0xd5]
diff --git a/llvm/test/MC/AArch64/arm64e-authenticated-pointer-reloc.s b/llvm/test/MC/AArch64/arm64e-authenticated-pointer-reloc.s
new file mode 100644
index 000000000000000..04fa5098c4a3e2b
--- /dev/null
+++ b/llvm/test/MC/AArch64/arm64e-authenticated-pointer-reloc.s
@@ -0,0 +1,119 @@
+// RUN: llvm-mc -triple=arm64-apple-darwin < %s | \
+// RUN:   FileCheck %s --check-prefix=ASM
+
+// RUN: llvm-mc -triple=arm64-apple-darwin -filetype=obj < %s | \
+// RUN:   llvm-readobj --expand-relocs --sections \
+// RUN:   --section-relocations --section-data - | \
+// RUN:   FileCheck %s --check-prefix=RELOC
+
+
+
+// RELOC:    Sections [
+// RELOC-LABEL: Section {
+// RELOC-LABEL: Section {
+// RELOC-NEXT:   Index: 1
+// RELOC-NEXT:   Name: __const (5F 5F 63 6F 6E 73 74 00 00 00 00 00 00 00 00 00)
+// RELOC-NEXT:   Segment: __DATA (5F 5F 44 41 54 41 00 00 00 00 00 00 00 00 00 00)
+
+.section	__DATA,__const
+.p2align	3
+
+// RELOC-LABEL: Relocations [
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x70
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g 7
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x60
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g 6
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x50
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g5
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x40
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g4
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x30
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g3
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x20
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g2
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x10
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g1
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x0
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g0
+// RELOC-NEXT:   }
+// RELOC-NEXT: ]
+// RELOC-NEXT: SectionData (
+
+// RELOC-NEXT:   0000: 00000000 2A000080
+// ASM:          .quad _g0@AUTH(ia,42)
+.quad _g0@AUTH(ia,42)
+.quad 0
+
+// RELOC-NEXT:   0010: 00000000 00000280
+// ASM:          .quad _g1@AUTH(ib,0)
+.quad _g1@AUTH(ib,0)
+.quad 0
+
+// RELOC-NEXT:   0020: 00000000 05000580
+// ASM:          .quad _g2@AUTH(da,5,addr)
+.quad _g2@AUTH(da,5,addr)
+.quad 0
+
+// RELOC-NEXT:   0030: 00000000 FFFF0780
+// ASM:          .quad _g3@AUTH(db,65535,addr)
+.quad _g3@AUTH(db,0xffff,addr)
+.quad 0
+
+// RELOC-NEXT:   0040: 07000000 00000080
+// ASM:          .quad (_g4+7)@AUTH(ia,0)
+.quad (_g4 + 7)@AUTH(ia,0)
+.quad 0
+
+// RELOC-NEXT:   0050: FDFFFFFF 00DE0280
+// ASM:          .quad (_g5-3)@AUTH(ib,56832)
+.quad (_g5 - 3)@AUTH(ib,0xde00)
+.quad 0
+
+// RELOC-NEXT:   0060: 00000000 FF000780
+// ASM:          .quad "_g 6"@AUTH(db,255,addr)
+.quad "_g 6"@AUTH(db,0xff,addr)
+.quad 0
+
+// RELOC-NEXT:   0070: 07000000 10000080
+// ASM:          .quad ("_g 7"+7)@AUTH(ia,16)
+.quad ("_g 7" + 7)@AUTH(ia,16)
+.quad 0
diff --git a/llvm/test/MC/AArch64/armv8.5a-ssbs-error.s b/llvm/test/MC/AArch64/armv8.5a-ssbs-error.s
index a7c9f4c4fbb5c3f..cd5ab43046c798a 100644
--- a/llvm/test/MC/AArch64/armv8.5a-ssbs-error.s
+++ b/llvm/test/MC/AArch64/armv8.5a-ssbs-error.s
@@ -1,5 +1,5 @@
 // RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+ssbs  < %s 2>&1 | FileCheck %s
-// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+v8.5a < %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+v8.5a < %s 2>&1 | FileCheck %s --check-prefix=NOSPECID
 // RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=-ssbs  < %s 2>&1 | FileCheck %s --check-prefix=NOSPECID
 
 msr SSBS, #16
diff --git a/llvm/test/MC/AArch64/armv8.5a-ssbs.s b/llvm/test/MC/AArch64/armv8.5a-ssbs.s
index ec6670f8ecc34c3..2a8b7b000646ad4 100644
--- a/llvm/test/MC/AArch64/armv8.5a-ssbs.s
+++ b/llvm/test/MC/AArch64/armv8.5a-ssbs.s
@@ -1,5 +1,5 @@
 // RUN:     llvm-mc -triple aarch64 -show-encoding -mattr=+ssbs  < %s      | FileCheck %s
-// RUN:     llvm-mc -triple aarch64 -show-encoding -mattr=+v8.5a < %s      | FileCheck %s
+// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+v8.5a < %s 2>&1 | FileCheck %s --check-prefix=NOSPECID
 // RUN:     llvm-mc -triple aarch64 -show-encoding -mcpu=cortex-a65 < %s   | FileCheck %s
 // RUN:     llvm-mc -triple aarch64 -show-encoding -mcpu=cortex-a65ae < %s | FileCheck %s
 // RUN:     llvm-mc -triple aarch64 -show-encoding -mcpu=cortex-a76 < %s   | FileCheck %s
diff --git a/llvm/test/MC/AArch64/mapping-within-section.s b/llvm/test/MC/AArch64/mapping-within-section.s
index c84e3a4d2fe647e..791d63b1c65b890 100644
--- a/llvm/test/MC/AArch64/mapping-within-section.s
+++ b/llvm/test/MC/AArch64/mapping-within-section.s
@@ -1,4 +1,5 @@
-// RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s | llvm-nm --no-sort --special-syms - | FileCheck %s --match-full-lines
+# RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s -o %t
+# RUN: llvm-readelf -Ss %t | FileCheck %s
 
     .text
 // $x at 0x0000
@@ -22,9 +23,11 @@
 $d:
 $x:
 
-// CHECK:      0000000000000000 t $x
-// CHECK-NEXT: 0000000000000004 t $d
-// CHECK-NEXT: 0000000000000064 t $x
-// CHECK-NEXT: 0000000000000068 t $x
-// CHECK-NEXT: 0000000000000068 T $d
-// CHECK-NOT:  {{.}}
+# CHECK: [[#TEXT:]]] .text
+
+# CHECK:      1: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT [[#TEXT]] $x
+# CHECK-NEXT: 2: 0000000000000004     0 NOTYPE  LOCAL  DEFAULT [[#TEXT]] $d
+# CHECK-NEXT: 3: 0000000000000064     0 NOTYPE  LOCAL  DEFAULT [[#TEXT]] $x
+# CHECK-NEXT: 4: 0000000000000068     0 NOTYPE  LOCAL  DEFAULT [[#TEXT]] $x
+# CHECK-NEXT: 5: 0000000000000068     0 NOTYPE  GLOBAL DEFAULT [[#TEXT]] $d
+# CHECK-NOT:  {{.}}
diff --git a/llvm/test/MC/AMDGPU/amd_kernel_code_t.s b/llvm/test/MC/AMDGPU/amd_kernel_code_t.s
index 052ec0bfabb849d..507001a2657c5f5 100644
--- a/llvm/test/MC/AMDGPU/amd_kernel_code_t.s
+++ b/llvm/test/MC/AMDGPU/amd_kernel_code_t.s
@@ -131,17 +131,17 @@ unknown_workitem_private_segment_byte_size:
 	s_endpgm
 
 ; ASM-LABEL: unknown_granulated_workitem_vgpr_count:
-; ASM: granulated_workitem_vgpr_count = ((0&4294967232)|(unknown&63))&63
-; ASM: granulated_wavefront_sgpr_count = (((0&4294967232)|(unknown&63))>>6)&15
-; ASM: priority = (((0&4294967232)|(unknown&63))>>10)&3
-; ASM: float_mode = (((0&4294967232)|(unknown&63))>>12)&255
-; ASM: priv = (((0&4294967232)|(unknown&63))>>20)&1
-; ASM: enable_dx10_clamp = (((0&4294967232)|(unknown&63))>>21)&1
-; ASM: debug_mode = (((0&4294967232)|(unknown&63))>>22)&1
-; ASM: enable_ieee_mode = (((0&4294967232)|(unknown&63))>>23)&1
-; ASM: enable_wgp_mode = (((0&4294967232)|(unknown&63))>>29)&1
-; ASM: enable_mem_ordered = (((0&4294967232)|(unknown&63))>>30)&1
-; ASM: enable_fwd_progress = (((0&4294967232)|(unknown&63))>>31)&1
+; ASM: granulated_workitem_vgpr_count = (unknown&63)&63
+; ASM: granulated_wavefront_sgpr_count = 0
+; ASM: priority = 0
+; ASM: float_mode = 0
+; ASM: priv = 0
+; ASM: enable_dx10_clamp = 0
+; ASM: debug_mode = 0
+; ASM: enable_ieee_mode = 0
+; ASM: enable_wgp_mode = 0
+; ASM: enable_mem_ordered = 0
+; ASM: enable_fwd_progress = 0
 .section .unknown_granulated_workitem_vgpr_count
 unknown_granulated_workitem_vgpr_count:
 	.amd_kernel_code_t
@@ -150,17 +150,9 @@ unknown_granulated_workitem_vgpr_count:
 	s_endpgm
 
 ; ASM-LABEL: unknown_enable_sgpr_workgroup_id_x:
-; ASM: enable_sgpr_private_segment_wave_byte_offset = ((0&4294967167)|((unknown&1)<<7))&1
-; ASM: user_sgpr_count = (((0&4294967167)|((unknown&1)<<7))>>1)&31
-; ASM: enable_trap_handler = (((0&4294967167)|((unknown&1)<<7))>>6)&1
-; ASM: enable_sgpr_workgroup_id_x = (((0&4294967167)|((unknown&1)<<7))>>7)&1
-; ASM: enable_sgpr_workgroup_id_y = (((0&4294967167)|((unknown&1)<<7))>>8)&1
-; ASM: enable_sgpr_workgroup_id_z = (((0&4294967167)|((unknown&1)<<7))>>9)&1
-; ASM: enable_sgpr_workgroup_info = (((0&4294967167)|((unknown&1)<<7))>>10)&1
-; ASM: enable_vgpr_workitem_id = (((0&4294967167)|((unknown&1)<<7))>>11)&3
-; ASM: enable_exception_msb = (((0&4294967167)|((unknown&1)<<7))>>13)&3
-; ASM: granulated_lds_size = (((0&4294967167)|((unknown&1)<<7))>>15)&511
-; ASM: enable_exception = (((0&4294967167)|((unknown&1)<<7))>>24)&127
+; ASM: enable_sgpr_workgroup_id_x = (((unknown&1)<<7)>>7)&1
+; ASM: enable_sgpr_workgroup_id_y = 0
+; ASM: enable_sgpr_workgroup_id_z = 0
 .section .unknown_enable_sgpr_workgroup_id_x
 unknown_enable_sgpr_workgroup_id_x:
 	.amd_kernel_code_t
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
index d2fe7b3d8a95a35..bd640e786e4ac12 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
@@ -1,3608 +1,3608 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding -comment-column=0 %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding -comment-column=0 | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding -comment-column=0 %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding -comment-column=0 | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
 
 // this file will be converted to true16 format when more true16 instructions are supported
 
 v_bfrev_b32_e32 v5, v1
-// GFX12: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, v1 ; encoding: [0x01,0x71,0x0a,0x7e]
 
 v_bfrev_b32 v5, v255
-// GFX12: v_bfrev_b32_e32 v5, v255                ; encoding: [0xff,0x71,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, v255 ; encoding: [0xff,0x71,0x0a,0x7e]
 
 v_bfrev_b32 v5, s1
-// GFX12: v_bfrev_b32_e32 v5, s1                  ; encoding: [0x01,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, s1 ; encoding: [0x01,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, s105
-// GFX12: v_bfrev_b32_e32 v5, s105                ; encoding: [0x69,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, s105 ; encoding: [0x69,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, vcc_lo
-// GFX12: v_bfrev_b32_e32 v5, vcc_lo              ; encoding: [0x6a,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, vcc_hi
-// GFX12: v_bfrev_b32_e32 v5, vcc_hi              ; encoding: [0x6b,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, ttmp15
-// GFX12: v_bfrev_b32_e32 v5, ttmp15              ; encoding: [0x7b,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, m0
-// GFX12: v_bfrev_b32_e32 v5, m0                  ; encoding: [0x7d,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, m0 ; encoding: [0x7d,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, exec_lo
-// GFX12: v_bfrev_b32_e32 v5, exec_lo             ; encoding: [0x7e,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, exec_lo ; encoding: [0x7e,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, exec_hi
-// GFX12: v_bfrev_b32_e32 v5, exec_hi             ; encoding: [0x7f,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, exec_hi ; encoding: [0x7f,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, null
-// GFX12: v_bfrev_b32_e32 v5, null                ; encoding: [0x7c,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, null ; encoding: [0x7c,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, -1
-// GFX12: v_bfrev_b32_e32 v5, -1                  ; encoding: [0xc1,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, -1 ; encoding: [0xc1,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, 0.5
-// GFX12: v_bfrev_b32_e32 v5, 0.5                 ; encoding: [0xf0,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, 0.5 ; encoding: [0xf0,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, src_scc
-// GFX12: v_bfrev_b32_e32 v5, src_scc             ; encoding: [0xfd,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, src_scc ; encoding: [0xfd,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v255, 0xaf123456
-// GFX12: v_bfrev_b32_e32 v255, 0xaf123456        ; encoding: [0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_bfrev_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ceil_f16 v5, v1
-// GFX12: v_ceil_f16_e32 v5, v1                   ; encoding: [0x01,0xb9,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, v1 ; encoding: [0x01,0xb9,0x0a,0x7e]
 
 v_ceil_f16 v5, v127
-// GFX12: v_ceil_f16_e32 v5, v127                 ; encoding: [0x7f,0xb9,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, v127 ; encoding: [0x7f,0xb9,0x0a,0x7e]
 
 v_ceil_f16 v5, s1
-// GFX12: v_ceil_f16_e32 v5, s1                   ; encoding: [0x01,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, s1 ; encoding: [0x01,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, s105
-// GFX12: v_ceil_f16_e32 v5, s105                 ; encoding: [0x69,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, s105 ; encoding: [0x69,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, vcc_lo
-// GFX12: v_ceil_f16_e32 v5, vcc_lo               ; encoding: [0x6a,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, vcc_hi
-// GFX12: v_ceil_f16_e32 v5, vcc_hi               ; encoding: [0x6b,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, ttmp15
-// GFX12: v_ceil_f16_e32 v5, ttmp15               ; encoding: [0x7b,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, m0
-// GFX12: v_ceil_f16_e32 v5, m0                   ; encoding: [0x7d,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, m0 ; encoding: [0x7d,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, exec_lo
-// GFX12: v_ceil_f16_e32 v5, exec_lo              ; encoding: [0x7e,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, exec_hi
-// GFX12: v_ceil_f16_e32 v5, exec_hi              ; encoding: [0x7f,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, null
-// GFX12: v_ceil_f16_e32 v5, null                 ; encoding: [0x7c,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, null ; encoding: [0x7c,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, -1
-// GFX12: v_ceil_f16_e32 v5, -1                   ; encoding: [0xc1,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, -1 ; encoding: [0xc1,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, 0.5
-// GFX12: v_ceil_f16_e32 v5, 0.5                  ; encoding: [0xf0,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, src_scc
-// GFX12: v_ceil_f16_e32 v5, src_scc              ; encoding: [0xfd,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, src_scc ; encoding: [0xfd,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v127, 0xfe0b
-// GFX12: v_ceil_f16_e32 v127, 0xfe0b             ; encoding: [0xff,0xb8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_ceil_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_ceil_f32 v5, v1
-// GFX12: v_ceil_f32_e32 v5, v1                   ; encoding: [0x01,0x45,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, v1 ; encoding: [0x01,0x45,0x0a,0x7e]
 
 v_ceil_f32 v5, v255
-// GFX12: v_ceil_f32_e32 v5, v255                 ; encoding: [0xff,0x45,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, v255 ; encoding: [0xff,0x45,0x0a,0x7e]
 
 v_ceil_f32 v5, s1
-// GFX12: v_ceil_f32_e32 v5, s1                   ; encoding: [0x01,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, s1 ; encoding: [0x01,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, s105
-// GFX12: v_ceil_f32_e32 v5, s105                 ; encoding: [0x69,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, s105 ; encoding: [0x69,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, vcc_lo
-// GFX12: v_ceil_f32_e32 v5, vcc_lo               ; encoding: [0x6a,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, vcc_hi
-// GFX12: v_ceil_f32_e32 v5, vcc_hi               ; encoding: [0x6b,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, ttmp15
-// GFX12: v_ceil_f32_e32 v5, ttmp15               ; encoding: [0x7b,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, m0
-// GFX12: v_ceil_f32_e32 v5, m0                   ; encoding: [0x7d,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, m0 ; encoding: [0x7d,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, exec_lo
-// GFX12: v_ceil_f32_e32 v5, exec_lo              ; encoding: [0x7e,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, exec_lo ; encoding: [0x7e,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, exec_hi
-// GFX12: v_ceil_f32_e32 v5, exec_hi              ; encoding: [0x7f,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, exec_hi ; encoding: [0x7f,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, null
-// GFX12: v_ceil_f32_e32 v5, null                 ; encoding: [0x7c,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, null ; encoding: [0x7c,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, -1
-// GFX12: v_ceil_f32_e32 v5, -1                   ; encoding: [0xc1,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, -1 ; encoding: [0xc1,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, 0.5
-// GFX12: v_ceil_f32_e32 v5, 0.5                  ; encoding: [0xf0,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, 0.5 ; encoding: [0xf0,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, src_scc
-// GFX12: v_ceil_f32_e32 v5, src_scc              ; encoding: [0xfd,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, src_scc ; encoding: [0xfd,0x44,0x0a,0x7e]
 
 v_ceil_f32 v255, 0xaf123456
-// GFX12: v_ceil_f32_e32 v255, 0xaf123456         ; encoding: [0xff,0x44,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ceil_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x44,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ceil_f64 v[5:6], v[1:2]
-// GFX12: v_ceil_f64_e32 v[5:6], v[1:2]           ; encoding: [0x01,0x31,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x31,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], v[254:255]
-// GFX12: v_ceil_f64_e32 v[5:6], v[254:255]       ; encoding: [0xfe,0x31,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x31,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], s[2:3]
-// GFX12: v_ceil_f64_e32 v[5:6], s[2:3]           ; encoding: [0x02,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], s[104:105]
-// GFX12: v_ceil_f64_e32 v[5:6], s[104:105]       ; encoding: [0x68,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], vcc
-// GFX12: v_ceil_f64_e32 v[5:6], vcc              ; encoding: [0x6a,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], ttmp[14:15]
-// GFX12: v_ceil_f64_e32 v[5:6], ttmp[14:15]      ; encoding: [0x7a,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], exec
-// GFX12: v_ceil_f64_e32 v[5:6], exec             ; encoding: [0x7e,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], exec ; encoding: [0x7e,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], null
-// GFX12: v_ceil_f64_e32 v[5:6], null             ; encoding: [0x7c,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], null ; encoding: [0x7c,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], -1
-// GFX12: v_ceil_f64_e32 v[5:6], -1               ; encoding: [0xc1,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], 0.5
-// GFX12: v_ceil_f64_e32 v[5:6], 0.5              ; encoding: [0xf0,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], src_scc
-// GFX12: v_ceil_f64_e32 v[5:6], src_scc          ; encoding: [0xfd,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[254:255], 0xaf123456
-// GFX12: v_ceil_f64_e32 v[254:255], 0xaf123456   ; encoding: [0xff,0x30,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ceil_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x30,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cls_i32 v5, v1
-// GFX12: v_cls_i32_e32 v5, v1                    ; encoding: [0x01,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v1 ; encoding: [0x01,0x77,0x0a,0x7e]
 
 v_cls_i32 v5, v255
-// GFX12: v_cls_i32_e32 v5, v255                  ; encoding: [0xff,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v255 ; encoding: [0xff,0x77,0x0a,0x7e]
 
 v_cls_i32 v5, s1
-// GFX12: v_cls_i32_e32 v5, s1                    ; encoding: [0x01,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s1 ; encoding: [0x01,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, s105
-// GFX12: v_cls_i32_e32 v5, s105                  ; encoding: [0x69,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s105 ; encoding: [0x69,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, vcc_lo
-// GFX12: v_cls_i32_e32 v5, vcc_lo                ; encoding: [0x6a,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_lo ; encoding: [0x6a,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, vcc_hi
-// GFX12: v_cls_i32_e32 v5, vcc_hi                ; encoding: [0x6b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_hi ; encoding: [0x6b,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, ttmp15
-// GFX12: v_cls_i32_e32 v5, ttmp15                ; encoding: [0x7b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, ttmp15 ; encoding: [0x7b,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, m0
-// GFX12: v_cls_i32_e32 v5, m0                    ; encoding: [0x7d,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, m0 ; encoding: [0x7d,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, exec_lo
-// GFX12: v_cls_i32_e32 v5, exec_lo               ; encoding: [0x7e,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_lo ; encoding: [0x7e,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, exec_hi
-// GFX12: v_cls_i32_e32 v5, exec_hi               ; encoding: [0x7f,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_hi ; encoding: [0x7f,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, null
-// GFX12: v_cls_i32_e32 v5, null                  ; encoding: [0x7c,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, null ; encoding: [0x7c,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, -1
-// GFX12: v_cls_i32_e32 v5, -1                    ; encoding: [0xc1,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, -1 ; encoding: [0xc1,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, 0.5
-// GFX12: v_cls_i32_e32 v5, 0.5                   ; encoding: [0xf0,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, 0.5 ; encoding: [0xf0,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, src_scc
-// GFX12: v_cls_i32_e32 v5, src_scc               ; encoding: [0xfd,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, src_scc ; encoding: [0xfd,0x76,0x0a,0x7e]
 
 v_cls_i32 v255, 0xaf123456
-// GFX12: v_cls_i32_e32 v255, 0xaf123456          ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cls_i32_e32 v255, 0xaf123456 ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_clz_i32_u32 v5, v1
-// GFX12: v_clz_i32_u32_e32 v5, v1                ; encoding: [0x01,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v1 ; encoding: [0x01,0x73,0x0a,0x7e]
 
 v_clz_i32_u32 v5, v255
-// GFX12: v_clz_i32_u32_e32 v5, v255              ; encoding: [0xff,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v255 ; encoding: [0xff,0x73,0x0a,0x7e]
 
 v_clz_i32_u32 v5, s1
-// GFX12: v_clz_i32_u32_e32 v5, s1                ; encoding: [0x01,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s1 ; encoding: [0x01,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, s105
-// GFX12: v_clz_i32_u32_e32 v5, s105              ; encoding: [0x69,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s105 ; encoding: [0x69,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, vcc_lo
-// GFX12: v_clz_i32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_lo ; encoding: [0x6a,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, vcc_hi
-// GFX12: v_clz_i32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_hi ; encoding: [0x6b,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, ttmp15
-// GFX12: v_clz_i32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, ttmp15 ; encoding: [0x7b,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, m0
-// GFX12: v_clz_i32_u32_e32 v5, m0                ; encoding: [0x7d,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, m0 ; encoding: [0x7d,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, exec_lo
-// GFX12: v_clz_i32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_lo ; encoding: [0x7e,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, exec_hi
-// GFX12: v_clz_i32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_hi ; encoding: [0x7f,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, null
-// GFX12: v_clz_i32_u32_e32 v5, null              ; encoding: [0x7c,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, null ; encoding: [0x7c,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, -1
-// GFX12: v_clz_i32_u32_e32 v5, -1                ; encoding: [0xc1,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, -1 ; encoding: [0xc1,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, 0.5
-// GFX12: v_clz_i32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, 0.5 ; encoding: [0xf0,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, src_scc
-// GFX12: v_clz_i32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, src_scc ; encoding: [0xfd,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v255, 0xaf123456
-// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456 ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cos_f16 v5, v1
-// GFX12: v_cos_f16_e32 v5, v1                    ; encoding: [0x01,0xc3,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, v1 ; encoding: [0x01,0xc3,0x0a,0x7e]
 
 v_cos_f16 v5, v127
-// GFX12: v_cos_f16_e32 v5, v127                  ; encoding: [0x7f,0xc3,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, v127 ; encoding: [0x7f,0xc3,0x0a,0x7e]
 
 v_cos_f16 v5, s1
-// GFX12: v_cos_f16_e32 v5, s1                    ; encoding: [0x01,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, s1 ; encoding: [0x01,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, s105
-// GFX12: v_cos_f16_e32 v5, s105                  ; encoding: [0x69,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, s105 ; encoding: [0x69,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, vcc_lo
-// GFX12: v_cos_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, vcc_hi
-// GFX12: v_cos_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, ttmp15
-// GFX12: v_cos_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, m0
-// GFX12: v_cos_f16_e32 v5, m0                    ; encoding: [0x7d,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, m0 ; encoding: [0x7d,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, exec_lo
-// GFX12: v_cos_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, exec_lo ; encoding: [0x7e,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, exec_hi
-// GFX12: v_cos_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, exec_hi ; encoding: [0x7f,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, null
-// GFX12: v_cos_f16_e32 v5, null                  ; encoding: [0x7c,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, null ; encoding: [0x7c,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, -1
-// GFX12: v_cos_f16_e32 v5, -1                    ; encoding: [0xc1,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, -1 ; encoding: [0xc1,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, 0.5
-// GFX12: v_cos_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, 0.5 ; encoding: [0xf0,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, src_scc
-// GFX12: v_cos_f16_e32 v5, src_scc               ; encoding: [0xfd,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, src_scc ; encoding: [0xfd,0xc2,0x0a,0x7e]
 
 v_cos_f16 v127, 0xfe0b
-// GFX12: v_cos_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xc2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cos_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xc2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cos_f32 v5, v1
-// GFX12: v_cos_f32_e32 v5, v1                    ; encoding: [0x01,0x6d,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, v1 ; encoding: [0x01,0x6d,0x0a,0x7e]
 
 v_cos_f32 v5, v255
-// GFX12: v_cos_f32_e32 v5, v255                  ; encoding: [0xff,0x6d,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, v255 ; encoding: [0xff,0x6d,0x0a,0x7e]
 
 v_cos_f32 v5, s1
-// GFX12: v_cos_f32_e32 v5, s1                    ; encoding: [0x01,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, s1 ; encoding: [0x01,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, s105
-// GFX12: v_cos_f32_e32 v5, s105                  ; encoding: [0x69,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, s105 ; encoding: [0x69,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, vcc_lo
-// GFX12: v_cos_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, vcc_hi
-// GFX12: v_cos_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, ttmp15
-// GFX12: v_cos_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, m0
-// GFX12: v_cos_f32_e32 v5, m0                    ; encoding: [0x7d,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, m0 ; encoding: [0x7d,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, exec_lo
-// GFX12: v_cos_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, exec_lo ; encoding: [0x7e,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, exec_hi
-// GFX12: v_cos_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, exec_hi ; encoding: [0x7f,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, null
-// GFX12: v_cos_f32_e32 v5, null                  ; encoding: [0x7c,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, null ; encoding: [0x7c,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, -1
-// GFX12: v_cos_f32_e32 v5, -1                    ; encoding: [0xc1,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, -1 ; encoding: [0xc1,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, 0.5
-// GFX12: v_cos_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, 0.5 ; encoding: [0xf0,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, src_scc
-// GFX12: v_cos_f32_e32 v5, src_scc               ; encoding: [0xfd,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, src_scc ; encoding: [0xfd,0x6c,0x0a,0x7e]
 
 v_cos_f32 v255, 0xaf123456
-// GFX12: v_cos_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cos_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x6c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ctz_i32_b32 v5, v1
-// GFX12: v_ctz_i32_b32_e32 v5, v1                ; encoding: [0x01,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v1 ; encoding: [0x01,0x75,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, v255
-// GFX12: v_ctz_i32_b32_e32 v5, v255              ; encoding: [0xff,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v255 ; encoding: [0xff,0x75,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, s1
-// GFX12: v_ctz_i32_b32_e32 v5, s1                ; encoding: [0x01,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s1 ; encoding: [0x01,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, s105
-// GFX12: v_ctz_i32_b32_e32 v5, s105              ; encoding: [0x69,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s105 ; encoding: [0x69,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, vcc_lo
-// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, vcc_hi
-// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, ttmp15
-// GFX12: v_ctz_i32_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, m0
-// GFX12: v_ctz_i32_b32_e32 v5, m0                ; encoding: [0x7d,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, m0 ; encoding: [0x7d,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, exec_lo
-// GFX12: v_ctz_i32_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_lo ; encoding: [0x7e,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, exec_hi
-// GFX12: v_ctz_i32_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_hi ; encoding: [0x7f,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, null
-// GFX12: v_ctz_i32_b32_e32 v5, null              ; encoding: [0x7c,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, null ; encoding: [0x7c,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, -1
-// GFX12: v_ctz_i32_b32_e32 v5, -1                ; encoding: [0xc1,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, -1 ; encoding: [0xc1,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, 0.5
-// GFX12: v_ctz_i32_b32_e32 v5, 0.5               ; encoding: [0xf0,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, 0.5 ; encoding: [0xf0,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, src_scc
-// GFX12: v_ctz_i32_b32_e32 v5, src_scc           ; encoding: [0xfd,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, src_scc ; encoding: [0xfd,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v255, 0xaf123456
-// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_bf8_e32 v1, s3
-// GFX12: v_cvt_f32_bf8_e32 v1, s3                ; encoding: [0x03,0xda,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, s3 ; encoding: [0x03,0xda,0x02,0x7e]
 
 v_cvt_f32_bf8_e32 v1, 3
-// GFX12: v_cvt_f32_bf8_e32 v1, 3                 ; encoding: [0x83,0xda,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, 3 ; encoding: [0x83,0xda,0x02,0x7e]
 
 v_cvt_f32_bf8_e32 v1, v3
-// GFX12: v_cvt_f32_bf8_e32 v1, v3                ; encoding: [0x03,0xdb,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, v3 ; encoding: [0x03,0xdb,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, s3
-// GFX12: v_cvt_f32_fp8_e32 v1, s3                ; encoding: [0x03,0xd8,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, s3 ; encoding: [0x03,0xd8,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, 3
-// GFX12: v_cvt_f32_fp8_e32 v1, 3                 ; encoding: [0x83,0xd8,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, 3 ; encoding: [0x83,0xd8,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, v3
-// GFX12: v_cvt_f32_fp8_e32 v1, v3                ; encoding: [0x03,0xd9,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, v3 ; encoding: [0x03,0xd9,0x02,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], s3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], s3         ; encoding: [0x03,0xde,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], s3 ; encoding: [0x03,0xde,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], s5
-// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], s5         ; encoding: [0x05,0xde,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], s5 ; encoding: [0x05,0xde,0x06,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], 3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], 3          ; encoding: [0x83,0xde,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], 3 ; encoding: [0x83,0xde,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], 3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], 3          ; encoding: [0x83,0xde,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], 3 ; encoding: [0x83,0xde,0x06,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], v3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], v3         ; encoding: [0x03,0xdf,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], v3 ; encoding: [0x03,0xdf,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], v3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], v3         ; encoding: [0x03,0xdf,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], v3 ; encoding: [0x03,0xdf,0x06,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], s3
-// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], s3         ; encoding: [0x03,0xdc,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], s3 ; encoding: [0x03,0xdc,0x04,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], 3
-// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], 3          ; encoding: [0x83,0xdc,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], 3 ; encoding: [0x83,0xdc,0x04,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], v3
-// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], v3         ; encoding: [0x03,0xdd,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], v3 ; encoding: [0x03,0xdd,0x04,0x7e]
 
 v_cvt_f16_f32 v5, v1
-// GFX12: v_cvt_f16_f32_e32 v5, v1                ; encoding: [0x01,0x15,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, v1 ; encoding: [0x01,0x15,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, v255
-// GFX12: v_cvt_f16_f32_e32 v5, v255              ; encoding: [0xff,0x15,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, v255 ; encoding: [0xff,0x15,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, s1
-// GFX12: v_cvt_f16_f32_e32 v5, s1                ; encoding: [0x01,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, s1 ; encoding: [0x01,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, s105
-// GFX12: v_cvt_f16_f32_e32 v5, s105              ; encoding: [0x69,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, s105 ; encoding: [0x69,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, vcc_lo
-// GFX12: v_cvt_f16_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, vcc_hi
-// GFX12: v_cvt_f16_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, ttmp15
-// GFX12: v_cvt_f16_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, m0
-// GFX12: v_cvt_f16_f32_e32 v5, m0                ; encoding: [0x7d,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, m0 ; encoding: [0x7d,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, exec_lo
-// GFX12: v_cvt_f16_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, exec_lo ; encoding: [0x7e,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, exec_hi
-// GFX12: v_cvt_f16_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, exec_hi ; encoding: [0x7f,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, null
-// GFX12: v_cvt_f16_f32_e32 v5, null              ; encoding: [0x7c,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, null ; encoding: [0x7c,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, -1
-// GFX12: v_cvt_f16_f32_e32 v5, -1                ; encoding: [0xc1,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, -1 ; encoding: [0xc1,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, 0.5
-// GFX12: v_cvt_f16_f32_e32 v5, 0.5               ; encoding: [0xf0,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, 0.5 ; encoding: [0xf0,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, src_scc
-// GFX12: v_cvt_f16_f32_e32 v5, src_scc           ; encoding: [0xfd,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, src_scc ; encoding: [0xfd,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v127, 0xaf123456
-// GFX12: v_cvt_f16_f32_e32 v127, 0xaf123456      ; encoding: [0xff,0x14,0xfe,0x7e,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f16_f32_e32 v127, 0xaf123456 ; encoding: [0xff,0x14,0xfe,0x7e,0x56,0x34,0x12,0xaf]
 
 v_cvt_f16_i16 v5, v1
-// GFX12: v_cvt_f16_i16_e32 v5, v1                ; encoding: [0x01,0xa3,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, v1 ; encoding: [0x01,0xa3,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, v127
-// GFX12: v_cvt_f16_i16_e32 v5, v127              ; encoding: [0x7f,0xa3,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, v127 ; encoding: [0x7f,0xa3,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, s1
-// GFX12: v_cvt_f16_i16_e32 v5, s1                ; encoding: [0x01,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, s1 ; encoding: [0x01,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, s105
-// GFX12: v_cvt_f16_i16_e32 v5, s105              ; encoding: [0x69,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, s105 ; encoding: [0x69,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, vcc_lo
-// GFX12: v_cvt_f16_i16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, vcc_lo ; encoding: [0x6a,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, vcc_hi
-// GFX12: v_cvt_f16_i16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, vcc_hi ; encoding: [0x6b,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, ttmp15
-// GFX12: v_cvt_f16_i16_e32 v5, ttmp15            ; encoding: [0x7b,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, ttmp15 ; encoding: [0x7b,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, m0
-// GFX12: v_cvt_f16_i16_e32 v5, m0                ; encoding: [0x7d,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, m0 ; encoding: [0x7d,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, exec_lo
-// GFX12: v_cvt_f16_i16_e32 v5, exec_lo           ; encoding: [0x7e,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, exec_lo ; encoding: [0x7e,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, exec_hi
-// GFX12: v_cvt_f16_i16_e32 v5, exec_hi           ; encoding: [0x7f,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, exec_hi ; encoding: [0x7f,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, null
-// GFX12: v_cvt_f16_i16_e32 v5, null              ; encoding: [0x7c,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, null ; encoding: [0x7c,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, -1
-// GFX12: v_cvt_f16_i16_e32 v5, -1                ; encoding: [0xc1,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, -1 ; encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// GFX12-ASM: v_cvt_f16_i16_e32 v5, 0.5               ; encoding: [0xf0,0xa2,0x0a,0x7e]
-// GFX12-DIS: v_cvt_f16_i16_e32 v5, 0x3800            ; encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_cvt_f16_i16_e32 v5, 0.5 ; encoding: [0xf0,0xa2,0x0a,0x7e]
+// GFX12-DIS: v_cvt_f16_i16_e32 v5, 0x3800 ; encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_f16_i16 v5, src_scc
-// GFX12: v_cvt_f16_i16_e32 v5, src_scc           ; encoding: [0xfd,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, src_scc ; encoding: [0xfd,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v127, 0xfe0b
-// GFX12: v_cvt_f16_i16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f16_i16_e32 v127, 0xfe0b ; encoding: [0xff,0xa2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f16_u16 v5, v1
-// GFX12: v_cvt_f16_u16_e32 v5, v1                ; encoding: [0x01,0xa1,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, v1 ; encoding: [0x01,0xa1,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, v127
-// GFX12: v_cvt_f16_u16_e32 v5, v127              ; encoding: [0x7f,0xa1,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, v127 ; encoding: [0x7f,0xa1,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, s1
-// GFX12: v_cvt_f16_u16_e32 v5, s1                ; encoding: [0x01,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, s1 ; encoding: [0x01,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, s105
-// GFX12: v_cvt_f16_u16_e32 v5, s105              ; encoding: [0x69,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, s105 ; encoding: [0x69,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, vcc_lo
-// GFX12: v_cvt_f16_u16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, vcc_lo ; encoding: [0x6a,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, vcc_hi
-// GFX12: v_cvt_f16_u16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, vcc_hi ; encoding: [0x6b,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, ttmp15
-// GFX12: v_cvt_f16_u16_e32 v5, ttmp15            ; encoding: [0x7b,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, ttmp15 ; encoding: [0x7b,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, m0
-// GFX12: v_cvt_f16_u16_e32 v5, m0                ; encoding: [0x7d,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, m0 ; encoding: [0x7d,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, exec_lo
-// GFX12: v_cvt_f16_u16_e32 v5, exec_lo           ; encoding: [0x7e,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, exec_lo ; encoding: [0x7e,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, exec_hi
-// GFX12: v_cvt_f16_u16_e32 v5, exec_hi           ; encoding: [0x7f,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, exec_hi ; encoding: [0x7f,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, null
-// GFX12: v_cvt_f16_u16_e32 v5, null              ; encoding: [0x7c,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, null ; encoding: [0x7c,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, -1
-// GFX12: v_cvt_f16_u16_e32 v5, -1                ; encoding: [0xc1,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, -1 ; encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// GFX12-ASM: v_cvt_f16_u16_e32 v5, 0.5               ; encoding: [0xf0,0xa0,0x0a,0x7e]
-// GFX12-DIS: v_cvt_f16_u16_e32 v5, 0x3800            ; encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_cvt_f16_u16_e32 v5, 0.5 ; encoding: [0xf0,0xa0,0x0a,0x7e]
+// GFX12-DIS: v_cvt_f16_u16_e32 v5, 0x3800 ; encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_f16_u16 v5, src_scc
-// GFX12: v_cvt_f16_u16_e32 v5, src_scc           ; encoding: [0xfd,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, src_scc ; encoding: [0xfd,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v127, 0xfe0b
-// GFX12: v_cvt_f16_u16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f16_u16_e32 v127, 0xfe0b ; encoding: [0xff,0xa0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f32_f16 v5, v1
-// GFX12: v_cvt_f32_f16_e32 v5, v1                ; encoding: [0x01,0x17,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, v1 ; encoding: [0x01,0x17,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, v127
-// GFX12: v_cvt_f32_f16_e32 v5, v127              ; encoding: [0x7f,0x17,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, v127 ; encoding: [0x7f,0x17,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, s1
-// GFX12: v_cvt_f32_f16_e32 v5, s1                ; encoding: [0x01,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, s1 ; encoding: [0x01,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, s105
-// GFX12: v_cvt_f32_f16_e32 v5, s105              ; encoding: [0x69,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, s105 ; encoding: [0x69,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, vcc_lo
-// GFX12: v_cvt_f32_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, vcc_lo ; encoding: [0x6a,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, vcc_hi
-// GFX12: v_cvt_f32_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, vcc_hi ; encoding: [0x6b,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, ttmp15
-// GFX12: v_cvt_f32_f16_e32 v5, ttmp15            ; encoding: [0x7b,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, ttmp15 ; encoding: [0x7b,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, m0
-// GFX12: v_cvt_f32_f16_e32 v5, m0                ; encoding: [0x7d,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, m0 ; encoding: [0x7d,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, exec_lo
-// GFX12: v_cvt_f32_f16_e32 v5, exec_lo           ; encoding: [0x7e,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, exec_lo ; encoding: [0x7e,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, exec_hi
-// GFX12: v_cvt_f32_f16_e32 v5, exec_hi           ; encoding: [0x7f,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, exec_hi ; encoding: [0x7f,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, null
-// GFX12: v_cvt_f32_f16_e32 v5, null              ; encoding: [0x7c,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, null ; encoding: [0x7c,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, -1
-// GFX12: v_cvt_f32_f16_e32 v5, -1                ; encoding: [0xc1,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, -1 ; encoding: [0xc1,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, 0.5
-// GFX12: v_cvt_f32_f16_e32 v5, 0.5               ; encoding: [0xf0,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, 0.5 ; encoding: [0xf0,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, src_scc
-// GFX12: v_cvt_f32_f16_e32 v5, src_scc           ; encoding: [0xfd,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, src_scc ; encoding: [0xfd,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v255, 0xfe0b
-// GFX12: v_cvt_f32_f16_e32 v255, 0xfe0b          ; encoding: [0xff,0x16,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f32_f16_e32 v255, 0xfe0b ; encoding: [0xff,0x16,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f32_f64 v5, v[1:2]
-// GFX12: v_cvt_f32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x1f,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, v[1:2] ; encoding: [0x01,0x1f,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, v[254:255]
-// GFX12: v_cvt_f32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x1f,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, v[254:255] ; encoding: [0xfe,0x1f,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, s[2:3]
-// GFX12: v_cvt_f32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, s[2:3] ; encoding: [0x02,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, s[104:105]
-// GFX12: v_cvt_f32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, s[104:105] ; encoding: [0x68,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, vcc
-// GFX12: v_cvt_f32_f64_e32 v5, vcc               ; encoding: [0x6a,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, vcc ; encoding: [0x6a,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, ttmp[14:15]
-// GFX12: v_cvt_f32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, exec
-// GFX12: v_cvt_f32_f64_e32 v5, exec              ; encoding: [0x7e,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, exec ; encoding: [0x7e,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, null
-// GFX12: v_cvt_f32_f64_e32 v5, null              ; encoding: [0x7c,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, null ; encoding: [0x7c,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, -1
-// GFX12: v_cvt_f32_f64_e32 v5, -1                ; encoding: [0xc1,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, -1 ; encoding: [0xc1,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, 0.5
-// GFX12: v_cvt_f32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, 0.5 ; encoding: [0xf0,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, src_scc
-// GFX12: v_cvt_f32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, src_scc ; encoding: [0xfd,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v255, 0xaf123456
-// GFX12: v_cvt_f32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x1e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x1e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_i32 v5, v1
-// GFX12: v_cvt_f32_i32_e32 v5, v1                ; encoding: [0x01,0x0b,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, v1 ; encoding: [0x01,0x0b,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, v255
-// GFX12: v_cvt_f32_i32_e32 v5, v255              ; encoding: [0xff,0x0b,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, v255 ; encoding: [0xff,0x0b,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, s1
-// GFX12: v_cvt_f32_i32_e32 v5, s1                ; encoding: [0x01,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, s1 ; encoding: [0x01,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, s105
-// GFX12: v_cvt_f32_i32_e32 v5, s105              ; encoding: [0x69,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, s105 ; encoding: [0x69,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, vcc_lo
-// GFX12: v_cvt_f32_i32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, vcc_lo ; encoding: [0x6a,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, vcc_hi
-// GFX12: v_cvt_f32_i32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, vcc_hi ; encoding: [0x6b,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, ttmp15
-// GFX12: v_cvt_f32_i32_e32 v5, ttmp15            ; encoding: [0x7b,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, ttmp15 ; encoding: [0x7b,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, m0
-// GFX12: v_cvt_f32_i32_e32 v5, m0                ; encoding: [0x7d,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, m0 ; encoding: [0x7d,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, exec_lo
-// GFX12: v_cvt_f32_i32_e32 v5, exec_lo           ; encoding: [0x7e,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, exec_lo ; encoding: [0x7e,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, exec_hi
-// GFX12: v_cvt_f32_i32_e32 v5, exec_hi           ; encoding: [0x7f,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, exec_hi ; encoding: [0x7f,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, null
-// GFX12: v_cvt_f32_i32_e32 v5, null              ; encoding: [0x7c,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, null ; encoding: [0x7c,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, -1
-// GFX12: v_cvt_f32_i32_e32 v5, -1                ; encoding: [0xc1,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, -1 ; encoding: [0xc1,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, 0.5
-// GFX12: v_cvt_f32_i32_e32 v5, 0.5               ; encoding: [0xf0,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, 0.5 ; encoding: [0xf0,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, src_scc
-// GFX12: v_cvt_f32_i32_e32 v5, src_scc           ; encoding: [0xfd,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, src_scc ; encoding: [0xfd,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v255, 0xaf123456
-// GFX12: v_cvt_f32_i32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_i32_e32 v255, 0xaf123456 ; encoding: [0xff,0x0a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_u32 v5, v1
-// GFX12: v_cvt_f32_u32_e32 v5, v1                ; encoding: [0x01,0x0d,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, v1 ; encoding: [0x01,0x0d,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, v255
-// GFX12: v_cvt_f32_u32_e32 v5, v255              ; encoding: [0xff,0x0d,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, v255 ; encoding: [0xff,0x0d,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, s1
-// GFX12: v_cvt_f32_u32_e32 v5, s1                ; encoding: [0x01,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, s1 ; encoding: [0x01,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, s105
-// GFX12: v_cvt_f32_u32_e32 v5, s105              ; encoding: [0x69,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, s105 ; encoding: [0x69,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, vcc_lo
-// GFX12: v_cvt_f32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, vcc_lo ; encoding: [0x6a,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, vcc_hi
-// GFX12: v_cvt_f32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, vcc_hi ; encoding: [0x6b,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, ttmp15
-// GFX12: v_cvt_f32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, ttmp15 ; encoding: [0x7b,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, m0
-// GFX12: v_cvt_f32_u32_e32 v5, m0                ; encoding: [0x7d,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, m0 ; encoding: [0x7d,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, exec_lo
-// GFX12: v_cvt_f32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, exec_lo ; encoding: [0x7e,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, exec_hi
-// GFX12: v_cvt_f32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, exec_hi ; encoding: [0x7f,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, null
-// GFX12: v_cvt_f32_u32_e32 v5, null              ; encoding: [0x7c,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, null ; encoding: [0x7c,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, -1
-// GFX12: v_cvt_f32_u32_e32 v5, -1                ; encoding: [0xc1,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, -1 ; encoding: [0xc1,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, 0.5
-// GFX12: v_cvt_f32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, 0.5 ; encoding: [0xf0,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, src_scc
-// GFX12: v_cvt_f32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, src_scc ; encoding: [0xfd,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v255, 0xaf123456
-// GFX12: v_cvt_f32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_u32_e32 v255, 0xaf123456 ; encoding: [0xff,0x0c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte0 v5, v1
-// GFX12: v_cvt_f32_ubyte0_e32 v5, v1             ; encoding: [0x01,0x23,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, v1 ; encoding: [0x01,0x23,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, v255
-// GFX12: v_cvt_f32_ubyte0_e32 v5, v255           ; encoding: [0xff,0x23,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, v255 ; encoding: [0xff,0x23,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, s1
-// GFX12: v_cvt_f32_ubyte0_e32 v5, s1             ; encoding: [0x01,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, s1 ; encoding: [0x01,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, s105
-// GFX12: v_cvt_f32_ubyte0_e32 v5, s105           ; encoding: [0x69,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, s105 ; encoding: [0x69,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, vcc_lo
-// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_lo         ; encoding: [0x6a,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_lo ; encoding: [0x6a,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, vcc_hi
-// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_hi         ; encoding: [0x6b,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_hi ; encoding: [0x6b,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, ttmp15
-// GFX12: v_cvt_f32_ubyte0_e32 v5, ttmp15         ; encoding: [0x7b,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, ttmp15 ; encoding: [0x7b,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, m0
-// GFX12: v_cvt_f32_ubyte0_e32 v5, m0             ; encoding: [0x7d,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, m0 ; encoding: [0x7d,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, exec_lo
-// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_lo        ; encoding: [0x7e,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_lo ; encoding: [0x7e,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, exec_hi
-// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_hi        ; encoding: [0x7f,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_hi ; encoding: [0x7f,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, null
-// GFX12: v_cvt_f32_ubyte0_e32 v5, null           ; encoding: [0x7c,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, null ; encoding: [0x7c,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, -1
-// GFX12: v_cvt_f32_ubyte0_e32 v5, -1             ; encoding: [0xc1,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, -1 ; encoding: [0xc1,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, 0.5
-// GFX12: v_cvt_f32_ubyte0_e32 v5, 0.5            ; encoding: [0xf0,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, 0.5 ; encoding: [0xf0,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, src_scc
-// GFX12: v_cvt_f32_ubyte0_e32 v5, src_scc        ; encoding: [0xfd,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, src_scc ; encoding: [0xfd,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v255, 0xaf123456
-// GFX12: v_cvt_f32_ubyte0_e32 v255, 0xaf123456   ; encoding: [0xff,0x22,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte0_e32 v255, 0xaf123456 ; encoding: [0xff,0x22,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte1 v5, v1
-// GFX12: v_cvt_f32_ubyte1_e32 v5, v1             ; encoding: [0x01,0x25,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, v1 ; encoding: [0x01,0x25,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, v255
-// GFX12: v_cvt_f32_ubyte1_e32 v5, v255           ; encoding: [0xff,0x25,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, v255 ; encoding: [0xff,0x25,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, s1
-// GFX12: v_cvt_f32_ubyte1_e32 v5, s1             ; encoding: [0x01,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, s1 ; encoding: [0x01,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, s105
-// GFX12: v_cvt_f32_ubyte1_e32 v5, s105           ; encoding: [0x69,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, s105 ; encoding: [0x69,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, vcc_lo
-// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_lo         ; encoding: [0x6a,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_lo ; encoding: [0x6a,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, vcc_hi
-// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_hi         ; encoding: [0x6b,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_hi ; encoding: [0x6b,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, ttmp15
-// GFX12: v_cvt_f32_ubyte1_e32 v5, ttmp15         ; encoding: [0x7b,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, ttmp15 ; encoding: [0x7b,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, m0
-// GFX12: v_cvt_f32_ubyte1_e32 v5, m0             ; encoding: [0x7d,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, m0 ; encoding: [0x7d,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, exec_lo
-// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_lo        ; encoding: [0x7e,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_lo ; encoding: [0x7e,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, exec_hi
-// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_hi        ; encoding: [0x7f,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_hi ; encoding: [0x7f,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, null
-// GFX12: v_cvt_f32_ubyte1_e32 v5, null           ; encoding: [0x7c,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, null ; encoding: [0x7c,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, -1
-// GFX12: v_cvt_f32_ubyte1_e32 v5, -1             ; encoding: [0xc1,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, -1 ; encoding: [0xc1,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, 0.5
-// GFX12: v_cvt_f32_ubyte1_e32 v5, 0.5            ; encoding: [0xf0,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, 0.5 ; encoding: [0xf0,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, src_scc
-// GFX12: v_cvt_f32_ubyte1_e32 v5, src_scc        ; encoding: [0xfd,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, src_scc ; encoding: [0xfd,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v255, 0xaf123456
-// GFX12: v_cvt_f32_ubyte1_e32 v255, 0xaf123456   ; encoding: [0xff,0x24,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte1_e32 v255, 0xaf123456 ; encoding: [0xff,0x24,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte2 v5, v1
-// GFX12: v_cvt_f32_ubyte2_e32 v5, v1             ; encoding: [0x01,0x27,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, v1 ; encoding: [0x01,0x27,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, v255
-// GFX12: v_cvt_f32_ubyte2_e32 v5, v255           ; encoding: [0xff,0x27,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, v255 ; encoding: [0xff,0x27,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, s1
-// GFX12: v_cvt_f32_ubyte2_e32 v5, s1             ; encoding: [0x01,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, s1 ; encoding: [0x01,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, s105
-// GFX12: v_cvt_f32_ubyte2_e32 v5, s105           ; encoding: [0x69,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, s105 ; encoding: [0x69,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, vcc_lo
-// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_lo         ; encoding: [0x6a,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_lo ; encoding: [0x6a,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, vcc_hi
-// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_hi         ; encoding: [0x6b,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_hi ; encoding: [0x6b,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, ttmp15
-// GFX12: v_cvt_f32_ubyte2_e32 v5, ttmp15         ; encoding: [0x7b,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, ttmp15 ; encoding: [0x7b,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, m0
-// GFX12: v_cvt_f32_ubyte2_e32 v5, m0             ; encoding: [0x7d,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, m0 ; encoding: [0x7d,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, exec_lo
-// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_lo        ; encoding: [0x7e,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_lo ; encoding: [0x7e,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, exec_hi
-// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_hi        ; encoding: [0x7f,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_hi ; encoding: [0x7f,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, null
-// GFX12: v_cvt_f32_ubyte2_e32 v5, null           ; encoding: [0x7c,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, null ; encoding: [0x7c,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, -1
-// GFX12: v_cvt_f32_ubyte2_e32 v5, -1             ; encoding: [0xc1,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, -1 ; encoding: [0xc1,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, 0.5
-// GFX12: v_cvt_f32_ubyte2_e32 v5, 0.5            ; encoding: [0xf0,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, 0.5 ; encoding: [0xf0,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, src_scc
-// GFX12: v_cvt_f32_ubyte2_e32 v5, src_scc        ; encoding: [0xfd,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, src_scc ; encoding: [0xfd,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v255, 0xaf123456
-// GFX12: v_cvt_f32_ubyte2_e32 v255, 0xaf123456   ; encoding: [0xff,0x26,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte2_e32 v255, 0xaf123456 ; encoding: [0xff,0x26,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte3 v5, v1
-// GFX12: v_cvt_f32_ubyte3_e32 v5, v1             ; encoding: [0x01,0x29,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, v1 ; encoding: [0x01,0x29,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, v255
-// GFX12: v_cvt_f32_ubyte3_e32 v5, v255           ; encoding: [0xff,0x29,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, v255 ; encoding: [0xff,0x29,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, s1
-// GFX12: v_cvt_f32_ubyte3_e32 v5, s1             ; encoding: [0x01,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, s1 ; encoding: [0x01,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, s105
-// GFX12: v_cvt_f32_ubyte3_e32 v5, s105           ; encoding: [0x69,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, s105 ; encoding: [0x69,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, vcc_lo
-// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_lo         ; encoding: [0x6a,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_lo ; encoding: [0x6a,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, vcc_hi
-// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_hi         ; encoding: [0x6b,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_hi ; encoding: [0x6b,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, ttmp15
-// GFX12: v_cvt_f32_ubyte3_e32 v5, ttmp15         ; encoding: [0x7b,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, ttmp15 ; encoding: [0x7b,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, m0
-// GFX12: v_cvt_f32_ubyte3_e32 v5, m0             ; encoding: [0x7d,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, m0 ; encoding: [0x7d,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, exec_lo
-// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_lo        ; encoding: [0x7e,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_lo ; encoding: [0x7e,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, exec_hi
-// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_hi        ; encoding: [0x7f,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_hi ; encoding: [0x7f,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, null
-// GFX12: v_cvt_f32_ubyte3_e32 v5, null           ; encoding: [0x7c,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, null ; encoding: [0x7c,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, -1
-// GFX12: v_cvt_f32_ubyte3_e32 v5, -1             ; encoding: [0xc1,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, -1 ; encoding: [0xc1,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, 0.5
-// GFX12: v_cvt_f32_ubyte3_e32 v5, 0.5            ; encoding: [0xf0,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, 0.5 ; encoding: [0xf0,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, src_scc
-// GFX12: v_cvt_f32_ubyte3_e32 v5, src_scc        ; encoding: [0xfd,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, src_scc ; encoding: [0xfd,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v255, 0xaf123456
-// GFX12: v_cvt_f32_ubyte3_e32 v255, 0xaf123456   ; encoding: [0xff,0x28,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte3_e32 v255, 0xaf123456 ; encoding: [0xff,0x28,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_f32 v[5:6], v1
-// GFX12: v_cvt_f64_f32_e32 v[5:6], v1            ; encoding: [0x01,0x21,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], v1 ; encoding: [0x01,0x21,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], v255
-// GFX12: v_cvt_f64_f32_e32 v[5:6], v255          ; encoding: [0xff,0x21,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], v255 ; encoding: [0xff,0x21,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], s1
-// GFX12: v_cvt_f64_f32_e32 v[5:6], s1            ; encoding: [0x01,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], s1 ; encoding: [0x01,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], s105
-// GFX12: v_cvt_f64_f32_e32 v[5:6], s105          ; encoding: [0x69,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], s105 ; encoding: [0x69,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], vcc_lo
-// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_lo ; encoding: [0x6a,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], vcc_hi
-// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_hi ; encoding: [0x6b,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], ttmp15
-// GFX12: v_cvt_f64_f32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], ttmp15 ; encoding: [0x7b,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], m0
-// GFX12: v_cvt_f64_f32_e32 v[5:6], m0            ; encoding: [0x7d,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], m0 ; encoding: [0x7d,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], exec_lo
-// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_lo ; encoding: [0x7e,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], exec_hi
-// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_hi ; encoding: [0x7f,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], null
-// GFX12: v_cvt_f64_f32_e32 v[5:6], null          ; encoding: [0x7c,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], null ; encoding: [0x7c,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], -1
-// GFX12: v_cvt_f64_f32_e32 v[5:6], -1            ; encoding: [0xc1,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], -1 ; encoding: [0xc1,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], 0.5
-// GFX12: v_cvt_f64_f32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], 0.5 ; encoding: [0xf0,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], src_scc
-// GFX12: v_cvt_f64_f32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], src_scc ; encoding: [0xfd,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[254:255], 0xaf123456
 // GFX12: v_cvt_f64_f32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x20,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_i32 v[5:6], v1
-// GFX12: v_cvt_f64_i32_e32 v[5:6], v1            ; encoding: [0x01,0x09,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], v1 ; encoding: [0x01,0x09,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], v255
-// GFX12: v_cvt_f64_i32_e32 v[5:6], v255          ; encoding: [0xff,0x09,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], v255 ; encoding: [0xff,0x09,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], s1
-// GFX12: v_cvt_f64_i32_e32 v[5:6], s1            ; encoding: [0x01,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], s1 ; encoding: [0x01,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], s105
-// GFX12: v_cvt_f64_i32_e32 v[5:6], s105          ; encoding: [0x69,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], s105 ; encoding: [0x69,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], vcc_lo
-// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_lo ; encoding: [0x6a,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], vcc_hi
-// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_hi ; encoding: [0x6b,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], ttmp15
-// GFX12: v_cvt_f64_i32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], ttmp15 ; encoding: [0x7b,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], m0
-// GFX12: v_cvt_f64_i32_e32 v[5:6], m0            ; encoding: [0x7d,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], m0 ; encoding: [0x7d,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], exec_lo
-// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_lo ; encoding: [0x7e,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], exec_hi
-// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_hi ; encoding: [0x7f,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], null
-// GFX12: v_cvt_f64_i32_e32 v[5:6], null          ; encoding: [0x7c,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], null ; encoding: [0x7c,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], -1
-// GFX12: v_cvt_f64_i32_e32 v[5:6], -1            ; encoding: [0xc1,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], -1 ; encoding: [0xc1,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], 0.5
-// GFX12: v_cvt_f64_i32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], 0.5 ; encoding: [0xf0,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], src_scc
-// GFX12: v_cvt_f64_i32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], src_scc ; encoding: [0xfd,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[254:255], 0xaf123456
 // GFX12: v_cvt_f64_i32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x08,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_u32 v[5:6], v1
-// GFX12: v_cvt_f64_u32_e32 v[5:6], v1            ; encoding: [0x01,0x2d,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], v1 ; encoding: [0x01,0x2d,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], v255
-// GFX12: v_cvt_f64_u32_e32 v[5:6], v255          ; encoding: [0xff,0x2d,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], v255 ; encoding: [0xff,0x2d,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], s1
-// GFX12: v_cvt_f64_u32_e32 v[5:6], s1            ; encoding: [0x01,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], s1 ; encoding: [0x01,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], s105
-// GFX12: v_cvt_f64_u32_e32 v[5:6], s105          ; encoding: [0x69,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], s105 ; encoding: [0x69,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], vcc_lo
-// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_lo ; encoding: [0x6a,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], vcc_hi
-// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_hi ; encoding: [0x6b,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], ttmp15
-// GFX12: v_cvt_f64_u32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], ttmp15 ; encoding: [0x7b,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], m0
-// GFX12: v_cvt_f64_u32_e32 v[5:6], m0            ; encoding: [0x7d,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], m0 ; encoding: [0x7d,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], exec_lo
-// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_lo ; encoding: [0x7e,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], exec_hi
-// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_hi ; encoding: [0x7f,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], null
-// GFX12: v_cvt_f64_u32_e32 v[5:6], null          ; encoding: [0x7c,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], null ; encoding: [0x7c,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], -1
-// GFX12: v_cvt_f64_u32_e32 v[5:6], -1            ; encoding: [0xc1,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], -1 ; encoding: [0xc1,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], 0.5
-// GFX12: v_cvt_f64_u32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], 0.5 ; encoding: [0xf0,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], src_scc
-// GFX12: v_cvt_f64_u32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], src_scc ; encoding: [0xfd,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[254:255], 0xaf123456
 // GFX12: v_cvt_f64_u32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x2c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_floor_i32_f32 v5, v1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, v1          ; encoding: [0x01,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v1 ; encoding: [0x01,0x1b,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, v255
-// GFX12: v_cvt_floor_i32_f32_e32 v5, v255        ; encoding: [0xff,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v255 ; encoding: [0xff,0x1b,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, s1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, s1          ; encoding: [0x01,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s1 ; encoding: [0x01,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, s105
-// GFX12: v_cvt_floor_i32_f32_e32 v5, s105        ; encoding: [0x69,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s105 ; encoding: [0x69,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, ttmp15
-// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, m0
-// GFX12: v_cvt_floor_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, exec_lo
-// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, exec_hi
-// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, null
-// GFX12: v_cvt_floor_i32_f32_e32 v5, null        ; encoding: [0x7c,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, null ; encoding: [0x7c,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, -1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, 0.5
-// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, src_scc
-// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v255, 0xaf123456
 // GFX12: v_cvt_floor_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_flr_i32_f32 v5, v1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, v1          ; encoding: [0x01,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v1 ; encoding: [0x01,0x1b,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, v255
-// GFX12: v_cvt_floor_i32_f32_e32 v5, v255        ; encoding: [0xff,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v255 ; encoding: [0xff,0x1b,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, s1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, s1          ; encoding: [0x01,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s1 ; encoding: [0x01,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, s105
-// GFX12: v_cvt_floor_i32_f32_e32 v5, s105        ; encoding: [0x69,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s105 ; encoding: [0x69,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, ttmp15
-// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, m0
-// GFX12: v_cvt_floor_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, exec_lo
-// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, exec_hi
-// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, null
-// GFX12: v_cvt_floor_i32_f32_e32 v5, null        ; encoding: [0x7c,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, null ; encoding: [0x7c,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, -1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, 0.5
-// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, src_scc
-// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v255, 0xaf123456
 // GFX12: v_cvt_floor_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i16_f16 v5, v1
-// GFX12: v_cvt_i16_f16_e32 v5, v1                ; encoding: [0x01,0xa7,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, v1 ; encoding: [0x01,0xa7,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, v127
-// GFX12: v_cvt_i16_f16_e32 v5, v127              ; encoding: [0x7f,0xa7,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, v127 ; encoding: [0x7f,0xa7,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, s1
-// GFX12: v_cvt_i16_f16_e32 v5, s1                ; encoding: [0x01,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, s1 ; encoding: [0x01,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, s105
-// GFX12: v_cvt_i16_f16_e32 v5, s105              ; encoding: [0x69,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, s105 ; encoding: [0x69,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, vcc_lo
-// GFX12: v_cvt_i16_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, vcc_hi
-// GFX12: v_cvt_i16_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, ttmp15
-// GFX12: v_cvt_i16_f16_e32 v5, ttmp15            ; encoding: [0x7b,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, m0
-// GFX12: v_cvt_i16_f16_e32 v5, m0                ; encoding: [0x7d,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, m0 ; encoding: [0x7d,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, exec_lo
-// GFX12: v_cvt_i16_f16_e32 v5, exec_lo           ; encoding: [0x7e,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, exec_hi
-// GFX12: v_cvt_i16_f16_e32 v5, exec_hi           ; encoding: [0x7f,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, null
-// GFX12: v_cvt_i16_f16_e32 v5, null              ; encoding: [0x7c,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, null ; encoding: [0x7c,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, -1
-// GFX12: v_cvt_i16_f16_e32 v5, -1                ; encoding: [0xc1,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, -1 ; encoding: [0xc1,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, 0.5
-// GFX12: v_cvt_i16_f16_e32 v5, 0.5               ; encoding: [0xf0,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, src_scc
-// GFX12: v_cvt_i16_f16_e32 v5, src_scc           ; encoding: [0xfd,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, src_scc ; encoding: [0xfd,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v127, 0xfe0b
-// GFX12: v_cvt_i16_f16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_i16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xa6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_i32_f32 v5, v1
-// GFX12: v_cvt_i32_f32_e32 v5, v1                ; encoding: [0x01,0x11,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, v1 ; encoding: [0x01,0x11,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, v255
-// GFX12: v_cvt_i32_f32_e32 v5, v255              ; encoding: [0xff,0x11,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, v255 ; encoding: [0xff,0x11,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, s1
-// GFX12: v_cvt_i32_f32_e32 v5, s1                ; encoding: [0x01,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, s1 ; encoding: [0x01,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, s105
-// GFX12: v_cvt_i32_f32_e32 v5, s105              ; encoding: [0x69,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, s105 ; encoding: [0x69,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_i32_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_i32_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, ttmp15
-// GFX12: v_cvt_i32_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, m0
-// GFX12: v_cvt_i32_f32_e32 v5, m0                ; encoding: [0x7d,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, exec_lo
-// GFX12: v_cvt_i32_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, exec_hi
-// GFX12: v_cvt_i32_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, null
-// GFX12: v_cvt_i32_f32_e32 v5, null              ; encoding: [0x7c,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, null ; encoding: [0x7c,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, -1
-// GFX12: v_cvt_i32_f32_e32 v5, -1                ; encoding: [0xc1,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, 0.5
-// GFX12: v_cvt_i32_f32_e32 v5, 0.5               ; encoding: [0xf0,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, src_scc
-// GFX12: v_cvt_i32_f32_e32 v5, src_scc           ; encoding: [0xfd,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v255, 0xaf123456
-// GFX12: v_cvt_i32_f32_e32 v255, 0xaf123456      ; encoding: [0xff,0x10,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x10,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i32_f64 v5, v[1:2]
-// GFX12: v_cvt_i32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x07,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, v[1:2] ; encoding: [0x01,0x07,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, v[254:255]
-// GFX12: v_cvt_i32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x07,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, v[254:255] ; encoding: [0xfe,0x07,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, s[2:3]
-// GFX12: v_cvt_i32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, s[2:3] ; encoding: [0x02,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, s[104:105]
-// GFX12: v_cvt_i32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, s[104:105] ; encoding: [0x68,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, vcc
-// GFX12: v_cvt_i32_f64_e32 v5, vcc               ; encoding: [0x6a,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, vcc ; encoding: [0x6a,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, ttmp[14:15]
-// GFX12: v_cvt_i32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, exec
-// GFX12: v_cvt_i32_f64_e32 v5, exec              ; encoding: [0x7e,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, exec ; encoding: [0x7e,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, null
-// GFX12: v_cvt_i32_f64_e32 v5, null              ; encoding: [0x7c,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, null ; encoding: [0x7c,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, -1
-// GFX12: v_cvt_i32_f64_e32 v5, -1                ; encoding: [0xc1,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, -1 ; encoding: [0xc1,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, 0.5
-// GFX12: v_cvt_i32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, 0.5 ; encoding: [0xf0,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, src_scc
-// GFX12: v_cvt_i32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, src_scc ; encoding: [0xfd,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v255, 0xaf123456
-// GFX12: v_cvt_i32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x06,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_i32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x06,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i32_i16 v5, v1
-// GFX12: v_cvt_i32_i16_e32 v5, v1                ; encoding: [0x01,0xd5,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, v1 ; encoding: [0x01,0xd5,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, v127
-// GFX12: v_cvt_i32_i16_e32 v5, v127              ; encoding: [0x7f,0xd5,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, v127 ; encoding: [0x7f,0xd5,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, s1
-// GFX12: v_cvt_i32_i16_e32 v5, s1                ; encoding: [0x01,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, s1 ; encoding: [0x01,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, s105
-// GFX12: v_cvt_i32_i16_e32 v5, s105              ; encoding: [0x69,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, s105 ; encoding: [0x69,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, vcc_lo
-// GFX12: v_cvt_i32_i16_e32 v5, vcc_lo            ; encoding: [0x6a,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, vcc_lo ; encoding: [0x6a,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, vcc_hi
-// GFX12: v_cvt_i32_i16_e32 v5, vcc_hi            ; encoding: [0x6b,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, vcc_hi ; encoding: [0x6b,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, ttmp15
-// GFX12: v_cvt_i32_i16_e32 v5, ttmp15            ; encoding: [0x7b,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, ttmp15 ; encoding: [0x7b,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, m0
-// GFX12: v_cvt_i32_i16_e32 v5, m0                ; encoding: [0x7d,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, m0 ; encoding: [0x7d,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, exec_lo
-// GFX12: v_cvt_i32_i16_e32 v5, exec_lo           ; encoding: [0x7e,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, exec_lo ; encoding: [0x7e,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, exec_hi
-// GFX12: v_cvt_i32_i16_e32 v5, exec_hi           ; encoding: [0x7f,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, exec_hi ; encoding: [0x7f,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, null
-// GFX12: v_cvt_i32_i16_e32 v5, null              ; encoding: [0x7c,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, null ; encoding: [0x7c,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, -1
-// GFX12: v_cvt_i32_i16_e32 v5, -1                ; encoding: [0xc1,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, -1 ; encoding: [0xc1,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, 0.5
-// GFX12-ASM: v_cvt_i32_i16_e32 v5, 0.5               ; encoding: [0xf0,0xd4,0x0a,0x7e]
-// GFX12-DIS: v_cvt_i32_i16_e32 v5, 0x3800            ; encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_cvt_i32_i16_e32 v5, 0.5 ; encoding: [0xf0,0xd4,0x0a,0x7e]
+// GFX12-DIS: v_cvt_i32_i16_e32 v5, 0x3800 ; encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_i32_i16 v5, src_scc
-// GFX12: v_cvt_i32_i16_e32 v5, src_scc           ; encoding: [0xfd,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, src_scc ; encoding: [0xfd,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v255, 0xfe0b
-// GFX12: v_cvt_i32_i16_e32 v255, 0xfe0b          ; encoding: [0xff,0xd4,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_i32_i16_e32 v255, 0xfe0b ; encoding: [0xff,0xd4,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_cvt_nearest_i32_f32 v5, v1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1        ; encoding: [0x01,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1 ; encoding: [0x01,0x19,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, v255
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255      ; encoding: [0xff,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255 ; encoding: [0xff,0x19,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, s1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1        ; encoding: [0x01,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1 ; encoding: [0x01,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, s105
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105      ; encoding: [0x69,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105 ; encoding: [0x69,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo    ; encoding: [0x6a,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi    ; encoding: [0x6b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, ttmp15
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15    ; encoding: [0x7b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, m0
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0        ; encoding: [0x7d,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, exec_lo
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo   ; encoding: [0x7e,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, exec_hi
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi   ; encoding: [0x7f,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, null
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, null      ; encoding: [0x7c,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, null ; encoding: [0x7c,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, -1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1        ; encoding: [0xc1,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, 0.5
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5       ; encoding: [0xf0,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, src_scc
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc   ; encoding: [0xfd,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v255, 0xaf123456
 // GFX12: v_cvt_nearest_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_norm_i16_f16 v5, v1
-// GFX12: v_cvt_norm_i16_f16_e32 v5, v1           ; encoding: [0x01,0xc7,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, v1 ; encoding: [0x01,0xc7,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, v127
-// GFX12: v_cvt_norm_i16_f16_e32 v5, v127         ; encoding: [0x7f,0xc7,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, v127 ; encoding: [0x7f,0xc7,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, s1
-// GFX12: v_cvt_norm_i16_f16_e32 v5, s1           ; encoding: [0x01,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, s1 ; encoding: [0x01,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, s105
-// GFX12: v_cvt_norm_i16_f16_e32 v5, s105         ; encoding: [0x69,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, s105 ; encoding: [0x69,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, vcc_lo
-// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_lo       ; encoding: [0x6a,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, vcc_hi
-// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_hi       ; encoding: [0x6b,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, ttmp15
-// GFX12: v_cvt_norm_i16_f16_e32 v5, ttmp15       ; encoding: [0x7b,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, m0
-// GFX12: v_cvt_norm_i16_f16_e32 v5, m0           ; encoding: [0x7d,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, m0 ; encoding: [0x7d,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, exec_lo
-// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_lo      ; encoding: [0x7e,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, exec_hi
-// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_hi      ; encoding: [0x7f,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, null
-// GFX12: v_cvt_norm_i16_f16_e32 v5, null         ; encoding: [0x7c,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, null ; encoding: [0x7c,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, -1
-// GFX12: v_cvt_norm_i16_f16_e32 v5, -1           ; encoding: [0xc1,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, -1 ; encoding: [0xc1,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, 0.5
-// GFX12: v_cvt_norm_i16_f16_e32 v5, 0.5          ; encoding: [0xf0,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, src_scc
-// GFX12: v_cvt_norm_i16_f16_e32 v5, src_scc      ; encoding: [0xfd,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, src_scc ; encoding: [0xfd,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v127, 0xfe0b
-// GFX12: v_cvt_norm_i16_f16_e32 v127, 0xfe0b     ; encoding: [0xff,0xc6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_norm_i16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xc6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_norm_u16_f16 v5, v1
-// GFX12: v_cvt_norm_u16_f16_e32 v5, v1           ; encoding: [0x01,0xc9,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, v1 ; encoding: [0x01,0xc9,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, v127
-// GFX12: v_cvt_norm_u16_f16_e32 v5, v127         ; encoding: [0x7f,0xc9,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, v127 ; encoding: [0x7f,0xc9,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, s1
-// GFX12: v_cvt_norm_u16_f16_e32 v5, s1           ; encoding: [0x01,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, s1 ; encoding: [0x01,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, s105
-// GFX12: v_cvt_norm_u16_f16_e32 v5, s105         ; encoding: [0x69,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, s105 ; encoding: [0x69,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, vcc_lo
-// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_lo       ; encoding: [0x6a,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, vcc_hi
-// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_hi       ; encoding: [0x6b,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, ttmp15
-// GFX12: v_cvt_norm_u16_f16_e32 v5, ttmp15       ; encoding: [0x7b,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, m0
-// GFX12: v_cvt_norm_u16_f16_e32 v5, m0           ; encoding: [0x7d,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, m0 ; encoding: [0x7d,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, exec_lo
-// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_lo      ; encoding: [0x7e,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, exec_hi
-// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_hi      ; encoding: [0x7f,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, null
-// GFX12: v_cvt_norm_u16_f16_e32 v5, null         ; encoding: [0x7c,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, null ; encoding: [0x7c,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, -1
-// GFX12: v_cvt_norm_u16_f16_e32 v5, -1           ; encoding: [0xc1,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, -1 ; encoding: [0xc1,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, 0.5
-// GFX12: v_cvt_norm_u16_f16_e32 v5, 0.5          ; encoding: [0xf0,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, src_scc
-// GFX12: v_cvt_norm_u16_f16_e32 v5, src_scc      ; encoding: [0xfd,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, src_scc ; encoding: [0xfd,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v127, 0xfe0b
-// GFX12: v_cvt_norm_u16_f16_e32 v127, 0xfe0b     ; encoding: [0xff,0xc8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_norm_u16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xc8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_off_f32_i4 v5, v1
-// GFX12: v_cvt_off_f32_i4_e32 v5, v1             ; encoding: [0x01,0x1d,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, v1 ; encoding: [0x01,0x1d,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, v255
-// GFX12: v_cvt_off_f32_i4_e32 v5, v255           ; encoding: [0xff,0x1d,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, v255 ; encoding: [0xff,0x1d,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, s1
-// GFX12: v_cvt_off_f32_i4_e32 v5, s1             ; encoding: [0x01,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, s1 ; encoding: [0x01,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, s105
-// GFX12: v_cvt_off_f32_i4_e32 v5, s105           ; encoding: [0x69,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, s105 ; encoding: [0x69,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, vcc_lo
-// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_lo         ; encoding: [0x6a,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_lo ; encoding: [0x6a,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, vcc_hi
-// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_hi         ; encoding: [0x6b,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_hi ; encoding: [0x6b,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, ttmp15
-// GFX12: v_cvt_off_f32_i4_e32 v5, ttmp15         ; encoding: [0x7b,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, ttmp15 ; encoding: [0x7b,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, m0
-// GFX12: v_cvt_off_f32_i4_e32 v5, m0             ; encoding: [0x7d,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, m0 ; encoding: [0x7d,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, exec_lo
-// GFX12: v_cvt_off_f32_i4_e32 v5, exec_lo        ; encoding: [0x7e,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, exec_lo ; encoding: [0x7e,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, exec_hi
-// GFX12: v_cvt_off_f32_i4_e32 v5, exec_hi        ; encoding: [0x7f,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, exec_hi ; encoding: [0x7f,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, null
-// GFX12: v_cvt_off_f32_i4_e32 v5, null           ; encoding: [0x7c,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, null ; encoding: [0x7c,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, -1
-// GFX12: v_cvt_off_f32_i4_e32 v5, -1             ; encoding: [0xc1,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, -1 ; encoding: [0xc1,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, 0.5
-// GFX12: v_cvt_off_f32_i4_e32 v5, 0.5            ; encoding: [0xf0,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, 0.5 ; encoding: [0xf0,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, src_scc
-// GFX12: v_cvt_off_f32_i4_e32 v5, src_scc        ; encoding: [0xfd,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, src_scc ; encoding: [0xfd,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v255, 0x4f
-// GFX12: v_cvt_off_f32_i4_e32 v255, 0x4f         ; encoding: [0xff,0x1c,0xfe,0x7f,0x4f,0x00,0x00,0x00]
+// GFX12: v_cvt_off_f32_i4_e32 v255, 0x4f ; encoding: [0xff,0x1c,0xfe,0x7f,0x4f,0x00,0x00,0x00]
 
 v_cvt_rpi_i32_f32 v5, v1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1        ; encoding: [0x01,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1 ; encoding: [0x01,0x19,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, v255
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255      ; encoding: [0xff,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255 ; encoding: [0xff,0x19,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, s1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1        ; encoding: [0x01,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1 ; encoding: [0x01,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, s105
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105      ; encoding: [0x69,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105 ; encoding: [0x69,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo    ; encoding: [0x6a,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi    ; encoding: [0x6b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, ttmp15
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15    ; encoding: [0x7b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, m0
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0        ; encoding: [0x7d,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, exec_lo
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo   ; encoding: [0x7e,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, exec_hi
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi   ; encoding: [0x7f,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, null
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, null      ; encoding: [0x7c,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, null ; encoding: [0x7c,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, -1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1        ; encoding: [0xc1,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, 0.5
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5       ; encoding: [0xf0,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, src_scc
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc   ; encoding: [0xfd,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v255, 0xaf123456
 // GFX12: v_cvt_nearest_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u16_f16 v5, v1
-// GFX12: v_cvt_u16_f16_e32 v5, v1                ; encoding: [0x01,0xa5,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, v1 ; encoding: [0x01,0xa5,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, v127
-// GFX12: v_cvt_u16_f16_e32 v5, v127              ; encoding: [0x7f,0xa5,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, v127 ; encoding: [0x7f,0xa5,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, s1
-// GFX12: v_cvt_u16_f16_e32 v5, s1                ; encoding: [0x01,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, s1 ; encoding: [0x01,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, s105
-// GFX12: v_cvt_u16_f16_e32 v5, s105              ; encoding: [0x69,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, s105 ; encoding: [0x69,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, vcc_lo
-// GFX12: v_cvt_u16_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, vcc_hi
-// GFX12: v_cvt_u16_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, ttmp15
-// GFX12: v_cvt_u16_f16_e32 v5, ttmp15            ; encoding: [0x7b,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, m0
-// GFX12: v_cvt_u16_f16_e32 v5, m0                ; encoding: [0x7d,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, m0 ; encoding: [0x7d,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, exec_lo
-// GFX12: v_cvt_u16_f16_e32 v5, exec_lo           ; encoding: [0x7e,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, exec_hi
-// GFX12: v_cvt_u16_f16_e32 v5, exec_hi           ; encoding: [0x7f,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, null
-// GFX12: v_cvt_u16_f16_e32 v5, null              ; encoding: [0x7c,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, null ; encoding: [0x7c,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, -1
-// GFX12: v_cvt_u16_f16_e32 v5, -1                ; encoding: [0xc1,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, -1 ; encoding: [0xc1,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, 0.5
-// GFX12: v_cvt_u16_f16_e32 v5, 0.5               ; encoding: [0xf0,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, src_scc
-// GFX12: v_cvt_u16_f16_e32 v5, src_scc           ; encoding: [0xfd,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, src_scc ; encoding: [0xfd,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v127, 0xfe0b
-// GFX12: v_cvt_u16_f16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_u16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xa4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_u32_f32 v5, v1
-// GFX12: v_cvt_u32_f32_e32 v5, v1                ; encoding: [0x01,0x0f,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, v1 ; encoding: [0x01,0x0f,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, v255
-// GFX12: v_cvt_u32_f32_e32 v5, v255              ; encoding: [0xff,0x0f,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, v255 ; encoding: [0xff,0x0f,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, s1
-// GFX12: v_cvt_u32_f32_e32 v5, s1                ; encoding: [0x01,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, s1 ; encoding: [0x01,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, s105
-// GFX12: v_cvt_u32_f32_e32 v5, s105              ; encoding: [0x69,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, s105 ; encoding: [0x69,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, vcc_lo
-// GFX12: v_cvt_u32_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, vcc_hi
-// GFX12: v_cvt_u32_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, ttmp15
-// GFX12: v_cvt_u32_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, m0
-// GFX12: v_cvt_u32_f32_e32 v5, m0                ; encoding: [0x7d,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, m0 ; encoding: [0x7d,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, exec_lo
-// GFX12: v_cvt_u32_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, exec_hi
-// GFX12: v_cvt_u32_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, null
-// GFX12: v_cvt_u32_f32_e32 v5, null              ; encoding: [0x7c,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, null ; encoding: [0x7c,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, -1
-// GFX12: v_cvt_u32_f32_e32 v5, -1                ; encoding: [0xc1,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, -1 ; encoding: [0xc1,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, 0.5
-// GFX12: v_cvt_u32_f32_e32 v5, 0.5               ; encoding: [0xf0,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, src_scc
-// GFX12: v_cvt_u32_f32_e32 v5, src_scc           ; encoding: [0xfd,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, src_scc ; encoding: [0xfd,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v255, 0xaf123456
-// GFX12: v_cvt_u32_f32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_u32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x0e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u32_f64 v5, v[1:2]
-// GFX12: v_cvt_u32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x2b,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, v[1:2] ; encoding: [0x01,0x2b,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, v[254:255]
-// GFX12: v_cvt_u32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x2b,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, v[254:255] ; encoding: [0xfe,0x2b,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, s[2:3]
-// GFX12: v_cvt_u32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, s[2:3] ; encoding: [0x02,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, s[104:105]
-// GFX12: v_cvt_u32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, s[104:105] ; encoding: [0x68,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, vcc
-// GFX12: v_cvt_u32_f64_e32 v5, vcc               ; encoding: [0x6a,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, vcc ; encoding: [0x6a,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, ttmp[14:15]
-// GFX12: v_cvt_u32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, exec
-// GFX12: v_cvt_u32_f64_e32 v5, exec              ; encoding: [0x7e,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, exec ; encoding: [0x7e,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, null
-// GFX12: v_cvt_u32_f64_e32 v5, null              ; encoding: [0x7c,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, null ; encoding: [0x7c,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, -1
-// GFX12: v_cvt_u32_f64_e32 v5, -1                ; encoding: [0xc1,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, -1 ; encoding: [0xc1,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, 0.5
-// GFX12: v_cvt_u32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, 0.5 ; encoding: [0xf0,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, src_scc
-// GFX12: v_cvt_u32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, src_scc ; encoding: [0xfd,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v255, 0xaf123456
-// GFX12: v_cvt_u32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x2a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_u32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x2a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u32_u16 v5, v1
-// GFX12: v_cvt_u32_u16_e32 v5, v1                ; encoding: [0x01,0xd7,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, v1 ; encoding: [0x01,0xd7,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, v127
-// GFX12: v_cvt_u32_u16_e32 v5, v127              ; encoding: [0x7f,0xd7,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, v127 ; encoding: [0x7f,0xd7,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, s1
-// GFX12: v_cvt_u32_u16_e32 v5, s1                ; encoding: [0x01,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, s1 ; encoding: [0x01,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, s105
-// GFX12: v_cvt_u32_u16_e32 v5, s105              ; encoding: [0x69,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, s105 ; encoding: [0x69,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, vcc_lo
-// GFX12: v_cvt_u32_u16_e32 v5, vcc_lo            ; encoding: [0x6a,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, vcc_lo ; encoding: [0x6a,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, vcc_hi
-// GFX12: v_cvt_u32_u16_e32 v5, vcc_hi            ; encoding: [0x6b,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, vcc_hi ; encoding: [0x6b,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, ttmp15
-// GFX12: v_cvt_u32_u16_e32 v5, ttmp15            ; encoding: [0x7b,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, ttmp15 ; encoding: [0x7b,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, m0
-// GFX12: v_cvt_u32_u16_e32 v5, m0                ; encoding: [0x7d,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, m0 ; encoding: [0x7d,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, exec_lo
-// GFX12: v_cvt_u32_u16_e32 v5, exec_lo           ; encoding: [0x7e,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, exec_lo ; encoding: [0x7e,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, exec_hi
-// GFX12: v_cvt_u32_u16_e32 v5, exec_hi           ; encoding: [0x7f,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, exec_hi ; encoding: [0x7f,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, null
-// GFX12: v_cvt_u32_u16_e32 v5, null              ; encoding: [0x7c,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, null ; encoding: [0x7c,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, -1
-// GFX12: v_cvt_u32_u16_e32 v5, -1                ; encoding: [0xc1,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, -1 ; encoding: [0xc1,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, 0.5
-// GFX12-ASM: v_cvt_u32_u16_e32 v5, 0.5               ; encoding: [0xf0,0xd6,0x0a,0x7e]
-// GFX12-DIS: v_cvt_u32_u16_e32 v5, 0x3800            ; encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_cvt_u32_u16_e32 v5, 0.5 ; encoding: [0xf0,0xd6,0x0a,0x7e]
+// GFX12-DIS: v_cvt_u32_u16_e32 v5, 0x3800 ; encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_u32_u16 v5, src_scc
-// GFX12: v_cvt_u32_u16_e32 v5, src_scc           ; encoding: [0xfd,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, src_scc ; encoding: [0xfd,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v255, 0xfe0b
-// GFX12: v_cvt_u32_u16_e32 v255, 0xfe0b          ; encoding: [0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_u32_u16_e32 v255, 0xfe0b ; encoding: [0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_exp_f16 v5, v1
-// GFX12: v_exp_f16_e32 v5, v1                    ; encoding: [0x01,0xb1,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, v1 ; encoding: [0x01,0xb1,0x0a,0x7e]
 
 v_exp_f16 v5, v127
-// GFX12: v_exp_f16_e32 v5, v127                  ; encoding: [0x7f,0xb1,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, v127 ; encoding: [0x7f,0xb1,0x0a,0x7e]
 
 v_exp_f16 v5, s1
-// GFX12: v_exp_f16_e32 v5, s1                    ; encoding: [0x01,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, s1 ; encoding: [0x01,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, s105
-// GFX12: v_exp_f16_e32 v5, s105                  ; encoding: [0x69,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, s105 ; encoding: [0x69,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, vcc_lo
-// GFX12: v_exp_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, vcc_hi
-// GFX12: v_exp_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, ttmp15
-// GFX12: v_exp_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, m0
-// GFX12: v_exp_f16_e32 v5, m0                    ; encoding: [0x7d,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, m0 ; encoding: [0x7d,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, exec_lo
-// GFX12: v_exp_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, exec_hi
-// GFX12: v_exp_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, null
-// GFX12: v_exp_f16_e32 v5, null                  ; encoding: [0x7c,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, null ; encoding: [0x7c,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, -1
-// GFX12: v_exp_f16_e32 v5, -1                    ; encoding: [0xc1,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, -1 ; encoding: [0xc1,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, 0.5
-// GFX12: v_exp_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, src_scc
-// GFX12: v_exp_f16_e32 v5, src_scc               ; encoding: [0xfd,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, src_scc ; encoding: [0xfd,0xb0,0x0a,0x7e]
 
 v_exp_f16 v127, 0xfe0b
-// GFX12: v_exp_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xb0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_exp_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_exp_f32 v5, v1
-// GFX12: v_exp_f32_e32 v5, v1                    ; encoding: [0x01,0x4b,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, v1 ; encoding: [0x01,0x4b,0x0a,0x7e]
 
 v_exp_f32 v5, v255
-// GFX12: v_exp_f32_e32 v5, v255                  ; encoding: [0xff,0x4b,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, v255 ; encoding: [0xff,0x4b,0x0a,0x7e]
 
 v_exp_f32 v5, s1
-// GFX12: v_exp_f32_e32 v5, s1                    ; encoding: [0x01,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, s1 ; encoding: [0x01,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, s105
-// GFX12: v_exp_f32_e32 v5, s105                  ; encoding: [0x69,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, s105 ; encoding: [0x69,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, vcc_lo
-// GFX12: v_exp_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, vcc_hi
-// GFX12: v_exp_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, ttmp15
-// GFX12: v_exp_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, m0
-// GFX12: v_exp_f32_e32 v5, m0                    ; encoding: [0x7d,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, m0 ; encoding: [0x7d,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, exec_lo
-// GFX12: v_exp_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, exec_lo ; encoding: [0x7e,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, exec_hi
-// GFX12: v_exp_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, exec_hi ; encoding: [0x7f,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, null
-// GFX12: v_exp_f32_e32 v5, null                  ; encoding: [0x7c,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, null ; encoding: [0x7c,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, -1
-// GFX12: v_exp_f32_e32 v5, -1                    ; encoding: [0xc1,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, -1 ; encoding: [0xc1,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, 0.5
-// GFX12: v_exp_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, 0.5 ; encoding: [0xf0,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, src_scc
-// GFX12: v_exp_f32_e32 v5, src_scc               ; encoding: [0xfd,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, src_scc ; encoding: [0xfd,0x4a,0x0a,0x7e]
 
 v_exp_f32 v255, 0xaf123456
-// GFX12: v_exp_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_exp_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbh_i32 v5, v1
-// GFX12: v_cls_i32_e32 v5, v1                    ; encoding: [0x01,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v1 ; encoding: [0x01,0x77,0x0a,0x7e]
 
 v_ffbh_i32 v5, v255
-// GFX12: v_cls_i32_e32 v5, v255                  ; encoding: [0xff,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v255 ; encoding: [0xff,0x77,0x0a,0x7e]
 
 v_ffbh_i32 v5, s1
-// GFX12: v_cls_i32_e32 v5, s1                    ; encoding: [0x01,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s1 ; encoding: [0x01,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, s105
-// GFX12: v_cls_i32_e32 v5, s105                  ; encoding: [0x69,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s105 ; encoding: [0x69,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, vcc_lo
-// GFX12: v_cls_i32_e32 v5, vcc_lo                ; encoding: [0x6a,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_lo ; encoding: [0x6a,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, vcc_hi
-// GFX12: v_cls_i32_e32 v5, vcc_hi                ; encoding: [0x6b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_hi ; encoding: [0x6b,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, ttmp15
-// GFX12: v_cls_i32_e32 v5, ttmp15                ; encoding: [0x7b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, ttmp15 ; encoding: [0x7b,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, m0
-// GFX12: v_cls_i32_e32 v5, m0                    ; encoding: [0x7d,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, m0 ; encoding: [0x7d,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, exec_lo
-// GFX12: v_cls_i32_e32 v5, exec_lo               ; encoding: [0x7e,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_lo ; encoding: [0x7e,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, exec_hi
-// GFX12: v_cls_i32_e32 v5, exec_hi               ; encoding: [0x7f,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_hi ; encoding: [0x7f,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, null
-// GFX12: v_cls_i32_e32 v5, null                  ; encoding: [0x7c,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, null ; encoding: [0x7c,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, -1
-// GFX12: v_cls_i32_e32 v5, -1                    ; encoding: [0xc1,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, -1 ; encoding: [0xc1,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, 0.5
-// GFX12: v_cls_i32_e32 v5, 0.5                   ; encoding: [0xf0,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, 0.5 ; encoding: [0xf0,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, src_scc
-// GFX12: v_cls_i32_e32 v5, src_scc               ; encoding: [0xfd,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, src_scc ; encoding: [0xfd,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v255, 0xaf123456
-// GFX12: v_cls_i32_e32 v255, 0xaf123456          ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cls_i32_e32 v255, 0xaf123456 ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbh_u32 v5, v1
-// GFX12: v_clz_i32_u32_e32 v5, v1                ; encoding: [0x01,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v1 ; encoding: [0x01,0x73,0x0a,0x7e]
 
 v_ffbh_u32 v5, v255
-// GFX12: v_clz_i32_u32_e32 v5, v255              ; encoding: [0xff,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v255 ; encoding: [0xff,0x73,0x0a,0x7e]
 
 v_ffbh_u32 v5, s1
-// GFX12: v_clz_i32_u32_e32 v5, s1                ; encoding: [0x01,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s1 ; encoding: [0x01,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, s105
-// GFX12: v_clz_i32_u32_e32 v5, s105              ; encoding: [0x69,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s105 ; encoding: [0x69,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, vcc_lo
-// GFX12: v_clz_i32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_lo ; encoding: [0x6a,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, vcc_hi
-// GFX12: v_clz_i32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_hi ; encoding: [0x6b,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, ttmp15
-// GFX12: v_clz_i32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, ttmp15 ; encoding: [0x7b,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, m0
-// GFX12: v_clz_i32_u32_e32 v5, m0                ; encoding: [0x7d,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, m0 ; encoding: [0x7d,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, exec_lo
-// GFX12: v_clz_i32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_lo ; encoding: [0x7e,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, exec_hi
-// GFX12: v_clz_i32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_hi ; encoding: [0x7f,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, null
-// GFX12: v_clz_i32_u32_e32 v5, null              ; encoding: [0x7c,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, null ; encoding: [0x7c,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, -1
-// GFX12: v_clz_i32_u32_e32 v5, -1                ; encoding: [0xc1,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, -1 ; encoding: [0xc1,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, 0.5
-// GFX12: v_clz_i32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, 0.5 ; encoding: [0xf0,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, src_scc
-// GFX12: v_clz_i32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, src_scc ; encoding: [0xfd,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v255, 0xaf123456
-// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456 ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbl_b32 v5, v1
-// GFX12: v_ctz_i32_b32_e32 v5, v1                ; encoding: [0x01,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v1 ; encoding: [0x01,0x75,0x0a,0x7e]
 
 v_ffbl_b32 v5, v255
-// GFX12: v_ctz_i32_b32_e32 v5, v255              ; encoding: [0xff,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v255 ; encoding: [0xff,0x75,0x0a,0x7e]
 
 v_ffbl_b32 v5, s1
-// GFX12: v_ctz_i32_b32_e32 v5, s1                ; encoding: [0x01,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s1 ; encoding: [0x01,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, s105
-// GFX12: v_ctz_i32_b32_e32 v5, s105              ; encoding: [0x69,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s105 ; encoding: [0x69,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, vcc_lo
-// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, vcc_hi
-// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, ttmp15
-// GFX12: v_ctz_i32_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, m0
-// GFX12: v_ctz_i32_b32_e32 v5, m0                ; encoding: [0x7d,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, m0 ; encoding: [0x7d,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, exec_lo
-// GFX12: v_ctz_i32_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_lo ; encoding: [0x7e,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, exec_hi
-// GFX12: v_ctz_i32_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_hi ; encoding: [0x7f,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, null
-// GFX12: v_ctz_i32_b32_e32 v5, null              ; encoding: [0x7c,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, null ; encoding: [0x7c,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, -1
-// GFX12: v_ctz_i32_b32_e32 v5, -1                ; encoding: [0xc1,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, -1 ; encoding: [0xc1,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, 0.5
-// GFX12: v_ctz_i32_b32_e32 v5, 0.5               ; encoding: [0xf0,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, 0.5 ; encoding: [0xf0,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, src_scc
-// GFX12: v_ctz_i32_b32_e32 v5, src_scc           ; encoding: [0xfd,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, src_scc ; encoding: [0xfd,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v255, 0xaf123456
-// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_floor_f16 v5, v1
-// GFX12: v_floor_f16_e32 v5, v1                  ; encoding: [0x01,0xb7,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, v1 ; encoding: [0x01,0xb7,0x0a,0x7e]
 
 v_floor_f16 v5, v127
-// GFX12: v_floor_f16_e32 v5, v127                ; encoding: [0x7f,0xb7,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, v127 ; encoding: [0x7f,0xb7,0x0a,0x7e]
 
 v_floor_f16 v5, s1
-// GFX12: v_floor_f16_e32 v5, s1                  ; encoding: [0x01,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, s1 ; encoding: [0x01,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, s105
-// GFX12: v_floor_f16_e32 v5, s105                ; encoding: [0x69,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, s105 ; encoding: [0x69,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, vcc_lo
-// GFX12: v_floor_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, vcc_hi
-// GFX12: v_floor_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, ttmp15
-// GFX12: v_floor_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, m0
-// GFX12: v_floor_f16_e32 v5, m0                  ; encoding: [0x7d,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, m0 ; encoding: [0x7d,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, exec_lo
-// GFX12: v_floor_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, exec_hi
-// GFX12: v_floor_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, null
-// GFX12: v_floor_f16_e32 v5, null                ; encoding: [0x7c,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, null ; encoding: [0x7c,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, -1
-// GFX12: v_floor_f16_e32 v5, -1                  ; encoding: [0xc1,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, -1 ; encoding: [0xc1,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, 0.5
-// GFX12: v_floor_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, src_scc
-// GFX12: v_floor_f16_e32 v5, src_scc             ; encoding: [0xfd,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, src_scc ; encoding: [0xfd,0xb6,0x0a,0x7e]
 
 v_floor_f16 v127, 0xfe0b
-// GFX12: v_floor_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xb6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_floor_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_floor_f32 v5, v1
-// GFX12: v_floor_f32_e32 v5, v1                  ; encoding: [0x01,0x49,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, v1 ; encoding: [0x01,0x49,0x0a,0x7e]
 
 v_floor_f32 v5, v255
-// GFX12: v_floor_f32_e32 v5, v255                ; encoding: [0xff,0x49,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, v255 ; encoding: [0xff,0x49,0x0a,0x7e]
 
 v_floor_f32 v5, s1
-// GFX12: v_floor_f32_e32 v5, s1                  ; encoding: [0x01,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, s1 ; encoding: [0x01,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, s105
-// GFX12: v_floor_f32_e32 v5, s105                ; encoding: [0x69,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, s105 ; encoding: [0x69,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, vcc_lo
-// GFX12: v_floor_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, vcc_hi
-// GFX12: v_floor_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, ttmp15
-// GFX12: v_floor_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, m0
-// GFX12: v_floor_f32_e32 v5, m0                  ; encoding: [0x7d,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, m0 ; encoding: [0x7d,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, exec_lo
-// GFX12: v_floor_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, exec_lo ; encoding: [0x7e,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, exec_hi
-// GFX12: v_floor_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, exec_hi ; encoding: [0x7f,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, null
-// GFX12: v_floor_f32_e32 v5, null                ; encoding: [0x7c,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, null ; encoding: [0x7c,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, -1
-// GFX12: v_floor_f32_e32 v5, -1                  ; encoding: [0xc1,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, -1 ; encoding: [0xc1,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, 0.5
-// GFX12: v_floor_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, 0.5 ; encoding: [0xf0,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, src_scc
-// GFX12: v_floor_f32_e32 v5, src_scc             ; encoding: [0xfd,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, src_scc ; encoding: [0xfd,0x48,0x0a,0x7e]
 
 v_floor_f32 v255, 0xaf123456
-// GFX12: v_floor_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x48,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_floor_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x48,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_floor_f64 v[5:6], v[1:2]
-// GFX12: v_floor_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x35,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x35,0x0a,0x7e]
 
 v_floor_f64 v[5:6], v[254:255]
-// GFX12: v_floor_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x35,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x35,0x0a,0x7e]
 
 v_floor_f64 v[5:6], s[2:3]
-// GFX12: v_floor_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], s[104:105]
-// GFX12: v_floor_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], vcc
-// GFX12: v_floor_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], ttmp[14:15]
-// GFX12: v_floor_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], exec
-// GFX12: v_floor_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], exec ; encoding: [0x7e,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], null
-// GFX12: v_floor_f64_e32 v[5:6], null            ; encoding: [0x7c,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], null ; encoding: [0x7c,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], -1
-// GFX12: v_floor_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], 0.5
-// GFX12: v_floor_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], src_scc
-// GFX12: v_floor_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x34,0x0a,0x7e]
 
 v_floor_f64 v[254:255], 0xaf123456
-// GFX12: v_floor_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x34,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_floor_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x34,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_fract_f16 v5, v1
-// GFX12: v_fract_f16_e32 v5, v1                  ; encoding: [0x01,0xbf,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, v1 ; encoding: [0x01,0xbf,0x0a,0x7e]
 
 v_fract_f16 v5, v127
-// GFX12: v_fract_f16_e32 v5, v127                ; encoding: [0x7f,0xbf,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, v127 ; encoding: [0x7f,0xbf,0x0a,0x7e]
 
 v_fract_f16 v5, s1
-// GFX12: v_fract_f16_e32 v5, s1                  ; encoding: [0x01,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, s1 ; encoding: [0x01,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, s105
-// GFX12: v_fract_f16_e32 v5, s105                ; encoding: [0x69,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, s105 ; encoding: [0x69,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, vcc_lo
-// GFX12: v_fract_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, vcc_hi
-// GFX12: v_fract_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, ttmp15
-// GFX12: v_fract_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, m0
-// GFX12: v_fract_f16_e32 v5, m0                  ; encoding: [0x7d,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, m0 ; encoding: [0x7d,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, exec_lo
-// GFX12: v_fract_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, exec_lo ; encoding: [0x7e,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, exec_hi
-// GFX12: v_fract_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, exec_hi ; encoding: [0x7f,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, null
-// GFX12: v_fract_f16_e32 v5, null                ; encoding: [0x7c,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, null ; encoding: [0x7c,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, -1
-// GFX12: v_fract_f16_e32 v5, -1                  ; encoding: [0xc1,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, -1 ; encoding: [0xc1,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, 0.5
-// GFX12: v_fract_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, 0.5 ; encoding: [0xf0,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, src_scc
-// GFX12: v_fract_f16_e32 v5, src_scc             ; encoding: [0xfd,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, src_scc ; encoding: [0xfd,0xbe,0x0a,0x7e]
 
 v_fract_f16 v127, 0xfe0b
-// GFX12: v_fract_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xbe,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_fract_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xbe,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_fract_f32 v5, v1
-// GFX12: v_fract_f32_e32 v5, v1                  ; encoding: [0x01,0x41,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, v1 ; encoding: [0x01,0x41,0x0a,0x7e]
 
 v_fract_f32 v5, v255
-// GFX12: v_fract_f32_e32 v5, v255                ; encoding: [0xff,0x41,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, v255 ; encoding: [0xff,0x41,0x0a,0x7e]
 
 v_fract_f32 v5, s1
-// GFX12: v_fract_f32_e32 v5, s1                  ; encoding: [0x01,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, s1 ; encoding: [0x01,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, s105
-// GFX12: v_fract_f32_e32 v5, s105                ; encoding: [0x69,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, s105 ; encoding: [0x69,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, vcc_lo
-// GFX12: v_fract_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, vcc_hi
-// GFX12: v_fract_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, ttmp15
-// GFX12: v_fract_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, m0
-// GFX12: v_fract_f32_e32 v5, m0                  ; encoding: [0x7d,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, m0 ; encoding: [0x7d,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, exec_lo
-// GFX12: v_fract_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, exec_lo ; encoding: [0x7e,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, exec_hi
-// GFX12: v_fract_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, exec_hi ; encoding: [0x7f,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, null
-// GFX12: v_fract_f32_e32 v5, null                ; encoding: [0x7c,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, null ; encoding: [0x7c,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, -1
-// GFX12: v_fract_f32_e32 v5, -1                  ; encoding: [0xc1,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, -1 ; encoding: [0xc1,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, 0.5
-// GFX12: v_fract_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, 0.5 ; encoding: [0xf0,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, src_scc
-// GFX12: v_fract_f32_e32 v5, src_scc             ; encoding: [0xfd,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, src_scc ; encoding: [0xfd,0x40,0x0a,0x7e]
 
 v_fract_f32 v255, 0xaf123456
-// GFX12: v_fract_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x40,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_fract_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x40,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_fract_f64 v[5:6], v[1:2]
-// GFX12: v_fract_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x7d,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x7d,0x0a,0x7e]
 
 v_fract_f64 v[5:6], v[254:255]
-// GFX12: v_fract_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x7d,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x7d,0x0a,0x7e]
 
 v_fract_f64 v[5:6], s[2:3]
-// GFX12: v_fract_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], s[104:105]
-// GFX12: v_fract_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], vcc
-// GFX12: v_fract_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], ttmp[14:15]
-// GFX12: v_fract_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], exec
-// GFX12: v_fract_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], exec ; encoding: [0x7e,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], null
-// GFX12: v_fract_f64_e32 v[5:6], null            ; encoding: [0x7c,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], null ; encoding: [0x7c,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], -1
-// GFX12: v_fract_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], 0.5
-// GFX12: v_fract_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], src_scc
-// GFX12: v_fract_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[254:255], 0xaf123456
-// GFX12: v_fract_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x7c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_fract_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x7c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_exp_i16_f16 v5, v1
-// GFX12: v_frexp_exp_i16_f16_e32 v5, v1          ; encoding: [0x01,0xb5,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, v1 ; encoding: [0x01,0xb5,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, v127
-// GFX12: v_frexp_exp_i16_f16_e32 v5, v127        ; encoding: [0x7f,0xb5,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, v127 ; encoding: [0x7f,0xb5,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, s1
-// GFX12: v_frexp_exp_i16_f16_e32 v5, s1          ; encoding: [0x01,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, s1 ; encoding: [0x01,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, s105
-// GFX12: v_frexp_exp_i16_f16_e32 v5, s105        ; encoding: [0x69,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, s105 ; encoding: [0x69,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, vcc_lo
-// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_lo      ; encoding: [0x6a,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, vcc_hi
-// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_hi      ; encoding: [0x6b,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, ttmp15
-// GFX12: v_frexp_exp_i16_f16_e32 v5, ttmp15      ; encoding: [0x7b,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, m0
-// GFX12: v_frexp_exp_i16_f16_e32 v5, m0          ; encoding: [0x7d,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, m0 ; encoding: [0x7d,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, exec_lo
-// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_lo     ; encoding: [0x7e,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, exec_hi
-// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_hi     ; encoding: [0x7f,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, null
-// GFX12: v_frexp_exp_i16_f16_e32 v5, null        ; encoding: [0x7c,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, null ; encoding: [0x7c,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, -1
-// GFX12: v_frexp_exp_i16_f16_e32 v5, -1          ; encoding: [0xc1,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, -1 ; encoding: [0xc1,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, 0.5
-// GFX12: v_frexp_exp_i16_f16_e32 v5, 0.5         ; encoding: [0xf0,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, src_scc
-// GFX12: v_frexp_exp_i16_f16_e32 v5, src_scc     ; encoding: [0xfd,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, src_scc ; encoding: [0xfd,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v127, 0xfe0b
-// GFX12: v_frexp_exp_i16_f16_e32 v127, 0xfe0b    ; encoding: [0xff,0xb4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_frexp_exp_i16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_frexp_exp_i32_f32 v5, v1
-// GFX12: v_frexp_exp_i32_f32_e32 v5, v1          ; encoding: [0x01,0x7f,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, v1 ; encoding: [0x01,0x7f,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, v255
-// GFX12: v_frexp_exp_i32_f32_e32 v5, v255        ; encoding: [0xff,0x7f,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, v255 ; encoding: [0xff,0x7f,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, s1
-// GFX12: v_frexp_exp_i32_f32_e32 v5, s1          ; encoding: [0x01,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, s1 ; encoding: [0x01,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, s105
-// GFX12: v_frexp_exp_i32_f32_e32 v5, s105        ; encoding: [0x69,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, s105 ; encoding: [0x69,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, vcc_lo
-// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, vcc_hi
-// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, ttmp15
-// GFX12: v_frexp_exp_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, m0
-// GFX12: v_frexp_exp_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, exec_lo
-// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, exec_hi
-// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, null
-// GFX12: v_frexp_exp_i32_f32_e32 v5, null        ; encoding: [0x7c,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, null ; encoding: [0x7c,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, -1
-// GFX12: v_frexp_exp_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, 0.5
-// GFX12: v_frexp_exp_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, src_scc
-// GFX12: v_frexp_exp_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v255, 0xaf123456
 // GFX12: v_frexp_exp_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x7e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_exp_i32_f64 v5, v[1:2]
-// GFX12: v_frexp_exp_i32_f64_e32 v5, v[1:2]      ; encoding: [0x01,0x79,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, v[1:2] ; encoding: [0x01,0x79,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, v[254:255]
-// GFX12: v_frexp_exp_i32_f64_e32 v5, v[254:255]  ; encoding: [0xfe,0x79,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, v[254:255] ; encoding: [0xfe,0x79,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, s[2:3]
-// GFX12: v_frexp_exp_i32_f64_e32 v5, s[2:3]      ; encoding: [0x02,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, s[2:3] ; encoding: [0x02,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, s[104:105]
-// GFX12: v_frexp_exp_i32_f64_e32 v5, s[104:105]  ; encoding: [0x68,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, s[104:105] ; encoding: [0x68,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, vcc
-// GFX12: v_frexp_exp_i32_f64_e32 v5, vcc         ; encoding: [0x6a,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, vcc ; encoding: [0x6a,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, ttmp[14:15]
 // GFX12: v_frexp_exp_i32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, exec
-// GFX12: v_frexp_exp_i32_f64_e32 v5, exec        ; encoding: [0x7e,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, exec ; encoding: [0x7e,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, null
-// GFX12: v_frexp_exp_i32_f64_e32 v5, null        ; encoding: [0x7c,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, null ; encoding: [0x7c,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, -1
-// GFX12: v_frexp_exp_i32_f64_e32 v5, -1          ; encoding: [0xc1,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, -1 ; encoding: [0xc1,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, 0.5
-// GFX12: v_frexp_exp_i32_f64_e32 v5, 0.5         ; encoding: [0xf0,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, 0.5 ; encoding: [0xf0,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, src_scc
-// GFX12: v_frexp_exp_i32_f64_e32 v5, src_scc     ; encoding: [0xfd,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, src_scc ; encoding: [0xfd,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v255, 0xaf123456
 // GFX12: v_frexp_exp_i32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x78,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_mant_f16 v5, v1
-// GFX12: v_frexp_mant_f16_e32 v5, v1             ; encoding: [0x01,0xb3,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, v1 ; encoding: [0x01,0xb3,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, v127
-// GFX12: v_frexp_mant_f16_e32 v5, v127           ; encoding: [0x7f,0xb3,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, v127 ; encoding: [0x7f,0xb3,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, s1
-// GFX12: v_frexp_mant_f16_e32 v5, s1             ; encoding: [0x01,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, s1 ; encoding: [0x01,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, s105
-// GFX12: v_frexp_mant_f16_e32 v5, s105           ; encoding: [0x69,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, s105 ; encoding: [0x69,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, vcc_lo
-// GFX12: v_frexp_mant_f16_e32 v5, vcc_lo         ; encoding: [0x6a,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, vcc_hi
-// GFX12: v_frexp_mant_f16_e32 v5, vcc_hi         ; encoding: [0x6b,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, ttmp15
-// GFX12: v_frexp_mant_f16_e32 v5, ttmp15         ; encoding: [0x7b,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, m0
-// GFX12: v_frexp_mant_f16_e32 v5, m0             ; encoding: [0x7d,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, m0 ; encoding: [0x7d,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, exec_lo
-// GFX12: v_frexp_mant_f16_e32 v5, exec_lo        ; encoding: [0x7e,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, exec_hi
-// GFX12: v_frexp_mant_f16_e32 v5, exec_hi        ; encoding: [0x7f,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, null
-// GFX12: v_frexp_mant_f16_e32 v5, null           ; encoding: [0x7c,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, null ; encoding: [0x7c,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, -1
-// GFX12: v_frexp_mant_f16_e32 v5, -1             ; encoding: [0xc1,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, -1 ; encoding: [0xc1,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, 0.5
-// GFX12: v_frexp_mant_f16_e32 v5, 0.5            ; encoding: [0xf0,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, src_scc
-// GFX12: v_frexp_mant_f16_e32 v5, src_scc        ; encoding: [0xfd,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, src_scc ; encoding: [0xfd,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v127, 0xfe0b
-// GFX12: v_frexp_mant_f16_e32 v127, 0xfe0b       ; encoding: [0xff,0xb2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_frexp_mant_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_frexp_mant_f32 v5, v1
-// GFX12: v_frexp_mant_f32_e32 v5, v1             ; encoding: [0x01,0x81,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, v1 ; encoding: [0x01,0x81,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, v255
-// GFX12: v_frexp_mant_f32_e32 v5, v255           ; encoding: [0xff,0x81,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, v255 ; encoding: [0xff,0x81,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, s1
-// GFX12: v_frexp_mant_f32_e32 v5, s1             ; encoding: [0x01,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, s1 ; encoding: [0x01,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, s105
-// GFX12: v_frexp_mant_f32_e32 v5, s105           ; encoding: [0x69,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, s105 ; encoding: [0x69,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, vcc_lo
-// GFX12: v_frexp_mant_f32_e32 v5, vcc_lo         ; encoding: [0x6a,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, vcc_hi
-// GFX12: v_frexp_mant_f32_e32 v5, vcc_hi         ; encoding: [0x6b,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, ttmp15
-// GFX12: v_frexp_mant_f32_e32 v5, ttmp15         ; encoding: [0x7b,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, m0
-// GFX12: v_frexp_mant_f32_e32 v5, m0             ; encoding: [0x7d,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, m0 ; encoding: [0x7d,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, exec_lo
-// GFX12: v_frexp_mant_f32_e32 v5, exec_lo        ; encoding: [0x7e,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, exec_lo ; encoding: [0x7e,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, exec_hi
-// GFX12: v_frexp_mant_f32_e32 v5, exec_hi        ; encoding: [0x7f,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, exec_hi ; encoding: [0x7f,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, null
-// GFX12: v_frexp_mant_f32_e32 v5, null           ; encoding: [0x7c,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, null ; encoding: [0x7c,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, -1
-// GFX12: v_frexp_mant_f32_e32 v5, -1             ; encoding: [0xc1,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, -1 ; encoding: [0xc1,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, 0.5
-// GFX12: v_frexp_mant_f32_e32 v5, 0.5            ; encoding: [0xf0,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, 0.5 ; encoding: [0xf0,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, src_scc
-// GFX12: v_frexp_mant_f32_e32 v5, src_scc        ; encoding: [0xfd,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, src_scc ; encoding: [0xfd,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v255, 0xaf123456
-// GFX12: v_frexp_mant_f32_e32 v255, 0xaf123456   ; encoding: [0xff,0x80,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_frexp_mant_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x80,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_mant_f64 v[5:6], v[1:2]
-// GFX12: v_frexp_mant_f64_e32 v[5:6], v[1:2]     ; encoding: [0x01,0x7b,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x7b,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], v[254:255]
 // GFX12: v_frexp_mant_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x7b,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], s[2:3]
-// GFX12: v_frexp_mant_f64_e32 v[5:6], s[2:3]     ; encoding: [0x02,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], s[104:105]
 // GFX12: v_frexp_mant_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], vcc
-// GFX12: v_frexp_mant_f64_e32 v[5:6], vcc        ; encoding: [0x6a,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], ttmp[14:15]
 // GFX12: v_frexp_mant_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], exec
-// GFX12: v_frexp_mant_f64_e32 v[5:6], exec       ; encoding: [0x7e,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], exec ; encoding: [0x7e,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], null
-// GFX12: v_frexp_mant_f64_e32 v[5:6], null       ; encoding: [0x7c,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], null ; encoding: [0x7c,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], -1
-// GFX12: v_frexp_mant_f64_e32 v[5:6], -1         ; encoding: [0xc1,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], 0.5
-// GFX12: v_frexp_mant_f64_e32 v[5:6], 0.5        ; encoding: [0xf0,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], src_scc
-// GFX12: v_frexp_mant_f64_e32 v[5:6], src_scc    ; encoding: [0xfd,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[254:255], 0xaf123456
 // GFX12: v_frexp_mant_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x7a,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_log_f16 v5, v1
-// GFX12: v_log_f16_e32 v5, v1                    ; encoding: [0x01,0xaf,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, v1 ; encoding: [0x01,0xaf,0x0a,0x7e]
 
 v_log_f16 v5, v127
-// GFX12: v_log_f16_e32 v5, v127                  ; encoding: [0x7f,0xaf,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, v127 ; encoding: [0x7f,0xaf,0x0a,0x7e]
 
 v_log_f16 v5, s1
-// GFX12: v_log_f16_e32 v5, s1                    ; encoding: [0x01,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, s1 ; encoding: [0x01,0xae,0x0a,0x7e]
 
 v_log_f16 v5, s105
-// GFX12: v_log_f16_e32 v5, s105                  ; encoding: [0x69,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, s105 ; encoding: [0x69,0xae,0x0a,0x7e]
 
 v_log_f16 v5, vcc_lo
-// GFX12: v_log_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xae,0x0a,0x7e]
 
 v_log_f16 v5, vcc_hi
-// GFX12: v_log_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xae,0x0a,0x7e]
 
 v_log_f16 v5, ttmp15
-// GFX12: v_log_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xae,0x0a,0x7e]
 
 v_log_f16 v5, m0
-// GFX12: v_log_f16_e32 v5, m0                    ; encoding: [0x7d,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, m0 ; encoding: [0x7d,0xae,0x0a,0x7e]
 
 v_log_f16 v5, exec_lo
-// GFX12: v_log_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, exec_lo ; encoding: [0x7e,0xae,0x0a,0x7e]
 
 v_log_f16 v5, exec_hi
-// GFX12: v_log_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, exec_hi ; encoding: [0x7f,0xae,0x0a,0x7e]
 
 v_log_f16 v5, null
-// GFX12: v_log_f16_e32 v5, null                  ; encoding: [0x7c,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, null ; encoding: [0x7c,0xae,0x0a,0x7e]
 
 v_log_f16 v5, -1
-// GFX12: v_log_f16_e32 v5, -1                    ; encoding: [0xc1,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, -1 ; encoding: [0xc1,0xae,0x0a,0x7e]
 
 v_log_f16 v5, 0.5
-// GFX12: v_log_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, 0.5 ; encoding: [0xf0,0xae,0x0a,0x7e]
 
 v_log_f16 v5, src_scc
-// GFX12: v_log_f16_e32 v5, src_scc               ; encoding: [0xfd,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, src_scc ; encoding: [0xfd,0xae,0x0a,0x7e]
 
 v_log_f16 v127, 0xfe0b
-// GFX12: v_log_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xae,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_log_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xae,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_log_f32 v5, v1
-// GFX12: v_log_f32_e32 v5, v1                    ; encoding: [0x01,0x4f,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, v1 ; encoding: [0x01,0x4f,0x0a,0x7e]
 
 v_log_f32 v5, v255
-// GFX12: v_log_f32_e32 v5, v255                  ; encoding: [0xff,0x4f,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, v255 ; encoding: [0xff,0x4f,0x0a,0x7e]
 
 v_log_f32 v5, s1
-// GFX12: v_log_f32_e32 v5, s1                    ; encoding: [0x01,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, s1 ; encoding: [0x01,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, s105
-// GFX12: v_log_f32_e32 v5, s105                  ; encoding: [0x69,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, s105 ; encoding: [0x69,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, vcc_lo
-// GFX12: v_log_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, vcc_hi
-// GFX12: v_log_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, ttmp15
-// GFX12: v_log_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, m0
-// GFX12: v_log_f32_e32 v5, m0                    ; encoding: [0x7d,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, m0 ; encoding: [0x7d,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, exec_lo
-// GFX12: v_log_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, exec_lo ; encoding: [0x7e,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, exec_hi
-// GFX12: v_log_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, exec_hi ; encoding: [0x7f,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, null
-// GFX12: v_log_f32_e32 v5, null                  ; encoding: [0x7c,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, null ; encoding: [0x7c,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, -1
-// GFX12: v_log_f32_e32 v5, -1                    ; encoding: [0xc1,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, -1 ; encoding: [0xc1,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, 0.5
-// GFX12: v_log_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, 0.5 ; encoding: [0xf0,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, src_scc
-// GFX12: v_log_f32_e32 v5, src_scc               ; encoding: [0xfd,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, src_scc ; encoding: [0xfd,0x4e,0x0a,0x7e]
 
 v_log_f32 v255, 0xaf123456
-// GFX12: v_log_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_log_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_mov_b32 v5, v1
-// GFX12: v_mov_b32_e32 v5, v1                    ; encoding: [0x01,0x03,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, v1 ; encoding: [0x01,0x03,0x0a,0x7e]
 
 v_mov_b32 v5, v255
-// GFX12: v_mov_b32_e32 v5, v255                  ; encoding: [0xff,0x03,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, v255 ; encoding: [0xff,0x03,0x0a,0x7e]
 
 v_mov_b32 v5, s1
-// GFX12: v_mov_b32_e32 v5, s1                    ; encoding: [0x01,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, s1 ; encoding: [0x01,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, s105
-// GFX12: v_mov_b32_e32 v5, s105                  ; encoding: [0x69,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, s105 ; encoding: [0x69,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, vcc_lo
-// GFX12: v_mov_b32_e32 v5, vcc_lo                ; encoding: [0x6a,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, vcc_hi
-// GFX12: v_mov_b32_e32 v5, vcc_hi                ; encoding: [0x6b,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, ttmp15
-// GFX12: v_mov_b32_e32 v5, ttmp15                ; encoding: [0x7b,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, m0
-// GFX12: v_mov_b32_e32 v5, m0                    ; encoding: [0x7d,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, m0 ; encoding: [0x7d,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, exec_lo
-// GFX12: v_mov_b32_e32 v5, exec_lo               ; encoding: [0x7e,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, exec_lo ; encoding: [0x7e,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, exec_hi
-// GFX12: v_mov_b32_e32 v5, exec_hi               ; encoding: [0x7f,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, exec_hi ; encoding: [0x7f,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, null
-// GFX12: v_mov_b32_e32 v5, null                  ; encoding: [0x7c,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, null ; encoding: [0x7c,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, -1
-// GFX12: v_mov_b32_e32 v5, -1                    ; encoding: [0xc1,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, -1 ; encoding: [0xc1,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, 0.5
-// GFX12: v_mov_b32_e32 v5, 0.5                   ; encoding: [0xf0,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, 0.5 ; encoding: [0xf0,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, src_scc
-// GFX12: v_mov_b32_e32 v5, src_scc               ; encoding: [0xfd,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, src_scc ; encoding: [0xfd,0x02,0x0a,0x7e]
 
 v_mov_b32 v255, 0xaf123456
-// GFX12: v_mov_b32_e32 v255, 0xaf123456          ; encoding: [0xff,0x02,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_mov_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x02,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_movreld_b32 v5, v1
-// GFX12: v_movreld_b32_e32 v5, v1                ; encoding: [0x01,0x85,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, v1 ; encoding: [0x01,0x85,0x0a,0x7e]
 
 v_movreld_b32 v5, v255
-// GFX12: v_movreld_b32_e32 v5, v255              ; encoding: [0xff,0x85,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, v255 ; encoding: [0xff,0x85,0x0a,0x7e]
 
 v_movreld_b32 v5, s1
-// GFX12: v_movreld_b32_e32 v5, s1                ; encoding: [0x01,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, s1 ; encoding: [0x01,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, s105
-// GFX12: v_movreld_b32_e32 v5, s105              ; encoding: [0x69,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, s105 ; encoding: [0x69,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, vcc_lo
-// GFX12: v_movreld_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, vcc_hi
-// GFX12: v_movreld_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, ttmp15
-// GFX12: v_movreld_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, m0
-// GFX12: v_movreld_b32_e32 v5, m0                ; encoding: [0x7d,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, m0 ; encoding: [0x7d,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, exec_lo
-// GFX12: v_movreld_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, exec_lo ; encoding: [0x7e,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, exec_hi
-// GFX12: v_movreld_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, exec_hi ; encoding: [0x7f,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, null
-// GFX12: v_movreld_b32_e32 v5, null              ; encoding: [0x7c,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, null ; encoding: [0x7c,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, -1
-// GFX12: v_movreld_b32_e32 v5, -1                ; encoding: [0xc1,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, -1 ; encoding: [0xc1,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, 0.5
-// GFX12: v_movreld_b32_e32 v5, 0.5               ; encoding: [0xf0,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, 0.5 ; encoding: [0xf0,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, src_scc
-// GFX12: v_movreld_b32_e32 v5, src_scc           ; encoding: [0xfd,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, src_scc ; encoding: [0xfd,0x84,0x0a,0x7e]
 
 v_movreld_b32 v255, 0xaf123456
-// GFX12: v_movreld_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x84,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_movreld_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x84,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_movrels_b32 v5, v1
-// GFX12: v_movrels_b32_e32 v5, v1                ; encoding: [0x01,0x87,0x0a,0x7e]
+// GFX12: v_movrels_b32_e32 v5, v1 ; encoding: [0x01,0x87,0x0a,0x7e]
 
 v_movrels_b32 v255, v255
-// GFX12: v_movrels_b32_e32 v255, v255            ; encoding: [0xff,0x87,0xfe,0x7f]
+// GFX12: v_movrels_b32_e32 v255, v255 ; encoding: [0xff,0x87,0xfe,0x7f]
 
 v_movrelsd_2_b32 v5, v1
-// GFX12: v_movrelsd_2_b32_e32 v5, v1             ; encoding: [0x01,0x91,0x0a,0x7e]
+// GFX12: v_movrelsd_2_b32_e32 v5, v1 ; encoding: [0x01,0x91,0x0a,0x7e]
 
 v_movrelsd_2_b32 v255, v255
-// GFX12: v_movrelsd_2_b32_e32 v255, v255         ; encoding: [0xff,0x91,0xfe,0x7f]
+// GFX12: v_movrelsd_2_b32_e32 v255, v255 ; encoding: [0xff,0x91,0xfe,0x7f]
 
 v_movrelsd_b32 v5, v1
-// GFX12: v_movrelsd_b32_e32 v5, v1               ; encoding: [0x01,0x89,0x0a,0x7e]
+// GFX12: v_movrelsd_b32_e32 v5, v1 ; encoding: [0x01,0x89,0x0a,0x7e]
 
 v_movrelsd_b32 v255, v255
-// GFX12: v_movrelsd_b32_e32 v255, v255           ; encoding: [0xff,0x89,0xfe,0x7f]
+// GFX12: v_movrelsd_b32_e32 v255, v255 ; encoding: [0xff,0x89,0xfe,0x7f]
 
 v_nop
-// GFX12: v_nop                                   ; encoding: [0x00,0x00,0x00,0x7e]
+// GFX12: v_nop ; encoding: [0x00,0x00,0x00,0x7e]
 
 v_not_b16 v5, v1
-// GFX12: v_not_b16_e32 v5, v1                    ; encoding: [0x01,0xd3,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, v1 ; encoding: [0x01,0xd3,0x0a,0x7e]
 
 v_not_b16 v5, v127
-// GFX12: v_not_b16_e32 v5, v127                  ; encoding: [0x7f,0xd3,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, v127 ; encoding: [0x7f,0xd3,0x0a,0x7e]
 
 v_not_b16 v5, s1
-// GFX12: v_not_b16_e32 v5, s1                    ; encoding: [0x01,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, s1 ; encoding: [0x01,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, s105
-// GFX12: v_not_b16_e32 v5, s105                  ; encoding: [0x69,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, s105 ; encoding: [0x69,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, vcc_lo
-// GFX12: v_not_b16_e32 v5, vcc_lo                ; encoding: [0x6a,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, vcc_lo ; encoding: [0x6a,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, vcc_hi
-// GFX12: v_not_b16_e32 v5, vcc_hi                ; encoding: [0x6b,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, vcc_hi ; encoding: [0x6b,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, ttmp15
-// GFX12: v_not_b16_e32 v5, ttmp15                ; encoding: [0x7b,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, ttmp15 ; encoding: [0x7b,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, m0
-// GFX12: v_not_b16_e32 v5, m0                    ; encoding: [0x7d,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, m0 ; encoding: [0x7d,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, exec_lo
-// GFX12: v_not_b16_e32 v5, exec_lo               ; encoding: [0x7e,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, exec_lo ; encoding: [0x7e,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, exec_hi
-// GFX12: v_not_b16_e32 v5, exec_hi               ; encoding: [0x7f,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, exec_hi ; encoding: [0x7f,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, null
-// GFX12: v_not_b16_e32 v5, null                  ; encoding: [0x7c,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, null ; encoding: [0x7c,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, -1
-// GFX12: v_not_b16_e32 v5, -1                    ; encoding: [0xc1,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, -1 ; encoding: [0xc1,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, 0.5
-// GFX12-ASM: v_not_b16_e32 v5, 0.5                   ; encoding: [0xf0,0xd2,0x0a,0x7e]
-// GFX12-DIS: v_not_b16_e32 v5, 0x3800                ; encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_not_b16_e32 v5, 0.5 ; encoding: [0xf0,0xd2,0x0a,0x7e]
+// GFX12-DIS: v_not_b16_e32 v5, 0x3800 ; encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_not_b16 v5, src_scc
-// GFX12: v_not_b16_e32 v5, src_scc               ; encoding: [0xfd,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, src_scc ; encoding: [0xfd,0xd2,0x0a,0x7e]
 
 v_not_b16 v127, 0xfe0b
-// GFX12: v_not_b16_e32 v127, 0xfe0b              ; encoding: [0xff,0xd2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_not_b16_e32 v127, 0xfe0b ; encoding: [0xff,0xd2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_not_b32 v5, v1
-// GFX12: v_not_b32_e32 v5, v1                    ; encoding: [0x01,0x6f,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, v1 ; encoding: [0x01,0x6f,0x0a,0x7e]
 
 v_not_b32 v5, v255
-// GFX12: v_not_b32_e32 v5, v255                  ; encoding: [0xff,0x6f,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, v255 ; encoding: [0xff,0x6f,0x0a,0x7e]
 
 v_not_b32 v5, s1
-// GFX12: v_not_b32_e32 v5, s1                    ; encoding: [0x01,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, s1 ; encoding: [0x01,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, s105
-// GFX12: v_not_b32_e32 v5, s105                  ; encoding: [0x69,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, s105 ; encoding: [0x69,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, vcc_lo
-// GFX12: v_not_b32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, vcc_hi
-// GFX12: v_not_b32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, ttmp15
-// GFX12: v_not_b32_e32 v5, ttmp15                ; encoding: [0x7b,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, m0
-// GFX12: v_not_b32_e32 v5, m0                    ; encoding: [0x7d,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, m0 ; encoding: [0x7d,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, exec_lo
-// GFX12: v_not_b32_e32 v5, exec_lo               ; encoding: [0x7e,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, exec_lo ; encoding: [0x7e,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, exec_hi
-// GFX12: v_not_b32_e32 v5, exec_hi               ; encoding: [0x7f,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, exec_hi ; encoding: [0x7f,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, null
-// GFX12: v_not_b32_e32 v5, null                  ; encoding: [0x7c,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, null ; encoding: [0x7c,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, -1
-// GFX12: v_not_b32_e32 v5, -1                    ; encoding: [0xc1,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, -1 ; encoding: [0xc1,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, 0.5
-// GFX12: v_not_b32_e32 v5, 0.5                   ; encoding: [0xf0,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, 0.5 ; encoding: [0xf0,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, src_scc
-// GFX12: v_not_b32_e32 v5, src_scc               ; encoding: [0xfd,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, src_scc ; encoding: [0xfd,0x6e,0x0a,0x7e]
 
 v_not_b32 v255, 0xaf123456
-// GFX12: v_not_b32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_not_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x6e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_permlane64_b32 v5, v1
-// GFX12: v_permlane64_b32 v5, v1                 ; encoding: [0x01,0xcf,0x0a,0x7e]
+// GFX12: v_permlane64_b32 v5, v1 ; encoding: [0x01,0xcf,0x0a,0x7e]
 
 v_permlane64_b32 v255, v255
-// GFX12: v_permlane64_b32 v255, v255             ; encoding: [0xff,0xcf,0xfe,0x7f]
+// GFX12: v_permlane64_b32 v255, v255 ; encoding: [0xff,0xcf,0xfe,0x7f]
 
 v_pipeflush
-// GFX12: v_pipeflush                             ; encoding: [0x00,0x36,0x00,0x7e]
+// GFX12: v_pipeflush ; encoding: [0x00,0x36,0x00,0x7e]
 
 v_rcp_f16 v5, v1
-// GFX12: v_rcp_f16_e32 v5, v1                    ; encoding: [0x01,0xa9,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, v1 ; encoding: [0x01,0xa9,0x0a,0x7e]
 
 v_rcp_f16 v5, v127
-// GFX12: v_rcp_f16_e32 v5, v127                  ; encoding: [0x7f,0xa9,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, v127 ; encoding: [0x7f,0xa9,0x0a,0x7e]
 
 v_rcp_f16 v5, s1
-// GFX12: v_rcp_f16_e32 v5, s1                    ; encoding: [0x01,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, s1 ; encoding: [0x01,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, s105
-// GFX12: v_rcp_f16_e32 v5, s105                  ; encoding: [0x69,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, s105 ; encoding: [0x69,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, vcc_lo
-// GFX12: v_rcp_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, vcc_hi
-// GFX12: v_rcp_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, ttmp15
-// GFX12: v_rcp_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, m0
-// GFX12: v_rcp_f16_e32 v5, m0                    ; encoding: [0x7d,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, m0 ; encoding: [0x7d,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, exec_lo
-// GFX12: v_rcp_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, exec_lo ; encoding: [0x7e,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, exec_hi
-// GFX12: v_rcp_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, exec_hi ; encoding: [0x7f,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, null
-// GFX12: v_rcp_f16_e32 v5, null                  ; encoding: [0x7c,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, null ; encoding: [0x7c,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, -1
-// GFX12: v_rcp_f16_e32 v5, -1                    ; encoding: [0xc1,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, -1 ; encoding: [0xc1,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, 0.5
-// GFX12: v_rcp_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, 0.5 ; encoding: [0xf0,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, src_scc
-// GFX12: v_rcp_f16_e32 v5, src_scc               ; encoding: [0xfd,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, src_scc ; encoding: [0xfd,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v127, 0xfe0b
-// GFX12: v_rcp_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xa8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rcp_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xa8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rcp_f32 v5, v1
-// GFX12: v_rcp_f32_e32 v5, v1                    ; encoding: [0x01,0x55,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, v1 ; encoding: [0x01,0x55,0x0a,0x7e]
 
 v_rcp_f32 v5, v255
-// GFX12: v_rcp_f32_e32 v5, v255                  ; encoding: [0xff,0x55,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, v255 ; encoding: [0xff,0x55,0x0a,0x7e]
 
 v_rcp_f32 v5, s1
-// GFX12: v_rcp_f32_e32 v5, s1                    ; encoding: [0x01,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, s1 ; encoding: [0x01,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, s105
-// GFX12: v_rcp_f32_e32 v5, s105                  ; encoding: [0x69,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, s105 ; encoding: [0x69,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, vcc_lo
-// GFX12: v_rcp_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, vcc_hi
-// GFX12: v_rcp_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, ttmp15
-// GFX12: v_rcp_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, m0
-// GFX12: v_rcp_f32_e32 v5, m0                    ; encoding: [0x7d,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, m0 ; encoding: [0x7d,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, exec_lo
-// GFX12: v_rcp_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, exec_lo ; encoding: [0x7e,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, exec_hi
-// GFX12: v_rcp_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, exec_hi ; encoding: [0x7f,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, null
-// GFX12: v_rcp_f32_e32 v5, null                  ; encoding: [0x7c,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, null ; encoding: [0x7c,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, -1
-// GFX12: v_rcp_f32_e32 v5, -1                    ; encoding: [0xc1,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, -1 ; encoding: [0xc1,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, 0.5
-// GFX12: v_rcp_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, 0.5 ; encoding: [0xf0,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, src_scc
-// GFX12: v_rcp_f32_e32 v5, src_scc               ; encoding: [0xfd,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, src_scc ; encoding: [0xfd,0x54,0x0a,0x7e]
 
 v_rcp_f32 v255, 0xaf123456
-// GFX12: v_rcp_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x54,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x54,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rcp_f64 v[5:6], v[1:2]
-// GFX12: v_rcp_f64_e32 v[5:6], v[1:2]            ; encoding: [0x01,0x5f,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x5f,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], v[254:255]
-// GFX12: v_rcp_f64_e32 v[5:6], v[254:255]        ; encoding: [0xfe,0x5f,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x5f,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], s[2:3]
-// GFX12: v_rcp_f64_e32 v[5:6], s[2:3]            ; encoding: [0x02,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], s[104:105]
-// GFX12: v_rcp_f64_e32 v[5:6], s[104:105]        ; encoding: [0x68,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], vcc
-// GFX12: v_rcp_f64_e32 v[5:6], vcc               ; encoding: [0x6a,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], ttmp[14:15]
-// GFX12: v_rcp_f64_e32 v[5:6], ttmp[14:15]       ; encoding: [0x7a,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], exec
-// GFX12: v_rcp_f64_e32 v[5:6], exec              ; encoding: [0x7e,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], exec ; encoding: [0x7e,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], null
-// GFX12: v_rcp_f64_e32 v[5:6], null              ; encoding: [0x7c,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], null ; encoding: [0x7c,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], -1
-// GFX12: v_rcp_f64_e32 v[5:6], -1                ; encoding: [0xc1,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], 0.5
-// GFX12: v_rcp_f64_e32 v[5:6], 0.5               ; encoding: [0xf0,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], src_scc
-// GFX12: v_rcp_f64_e32 v[5:6], src_scc           ; encoding: [0xfd,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[254:255], 0xaf123456
-// GFX12: v_rcp_f64_e32 v[254:255], 0xaf123456    ; encoding: [0xff,0x5e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x5e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rcp_iflag_f32 v5, v1
-// GFX12: v_rcp_iflag_f32_e32 v5, v1              ; encoding: [0x01,0x57,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, v1 ; encoding: [0x01,0x57,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, v255
-// GFX12: v_rcp_iflag_f32_e32 v5, v255            ; encoding: [0xff,0x57,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, v255 ; encoding: [0xff,0x57,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, s1
-// GFX12: v_rcp_iflag_f32_e32 v5, s1              ; encoding: [0x01,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, s1 ; encoding: [0x01,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, s105
-// GFX12: v_rcp_iflag_f32_e32 v5, s105            ; encoding: [0x69,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, s105 ; encoding: [0x69,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, vcc_lo
-// GFX12: v_rcp_iflag_f32_e32 v5, vcc_lo          ; encoding: [0x6a,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, vcc_hi
-// GFX12: v_rcp_iflag_f32_e32 v5, vcc_hi          ; encoding: [0x6b,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, ttmp15
-// GFX12: v_rcp_iflag_f32_e32 v5, ttmp15          ; encoding: [0x7b,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, m0
-// GFX12: v_rcp_iflag_f32_e32 v5, m0              ; encoding: [0x7d,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, m0 ; encoding: [0x7d,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, exec_lo
-// GFX12: v_rcp_iflag_f32_e32 v5, exec_lo         ; encoding: [0x7e,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, exec_lo ; encoding: [0x7e,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, exec_hi
-// GFX12: v_rcp_iflag_f32_e32 v5, exec_hi         ; encoding: [0x7f,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, exec_hi ; encoding: [0x7f,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, null
-// GFX12: v_rcp_iflag_f32_e32 v5, null            ; encoding: [0x7c,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, null ; encoding: [0x7c,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, -1
-// GFX12: v_rcp_iflag_f32_e32 v5, -1              ; encoding: [0xc1,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, -1 ; encoding: [0xc1,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, 0.5
-// GFX12: v_rcp_iflag_f32_e32 v5, 0.5             ; encoding: [0xf0,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, 0.5 ; encoding: [0xf0,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, src_scc
-// GFX12: v_rcp_iflag_f32_e32 v5, src_scc         ; encoding: [0xfd,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, src_scc ; encoding: [0xfd,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v255, 0xaf123456
-// GFX12: v_rcp_iflag_f32_e32 v255, 0xaf123456    ; encoding: [0xff,0x56,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_iflag_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x56,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_readfirstlane_b32 s5, v1
-// GFX12: v_readfirstlane_b32 s5, v1              ; encoding: [0x01,0x05,0x0a,0x7e]
+// GFX12: v_readfirstlane_b32 s5, v1 ; encoding: [0x01,0x05,0x0a,0x7e]
 
 v_readfirstlane_b32 s105, v1
-// GFX12: v_readfirstlane_b32 s105, v1            ; encoding: [0x01,0x05,0xd2,0x7e]
+// GFX12: v_readfirstlane_b32 s105, v1 ; encoding: [0x01,0x05,0xd2,0x7e]
 
 v_readfirstlane_b32 vcc_lo, v1
-// GFX12: v_readfirstlane_b32 vcc_lo, v1          ; encoding: [0x01,0x05,0xd4,0x7e]
+// GFX12: v_readfirstlane_b32 vcc_lo, v1 ; encoding: [0x01,0x05,0xd4,0x7e]
 
 v_readfirstlane_b32 vcc_hi, v1
-// GFX12: v_readfirstlane_b32 vcc_hi, v1          ; encoding: [0x01,0x05,0xd6,0x7e]
+// GFX12: v_readfirstlane_b32 vcc_hi, v1 ; encoding: [0x01,0x05,0xd6,0x7e]
 
 v_readfirstlane_b32 ttmp15, v1
-// GFX12: v_readfirstlane_b32 ttmp15, v1          ; encoding: [0x01,0x05,0xf6,0x7e]
+// GFX12: v_readfirstlane_b32 ttmp15, v1 ; encoding: [0x01,0x05,0xf6,0x7e]
 
 v_readfirstlane_b32 null, v255
-// GFX12: v_readfirstlane_b32 null, v255          ; encoding: [0xff,0x05,0xf8,0x7e]
+// GFX12: v_readfirstlane_b32 null, v255 ; encoding: [0xff,0x05,0xf8,0x7e]
 
 v_rndne_f16 v5, v1
-// GFX12: v_rndne_f16_e32 v5, v1                  ; encoding: [0x01,0xbd,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, v1 ; encoding: [0x01,0xbd,0x0a,0x7e]
 
 v_rndne_f16 v5, v127
-// GFX12: v_rndne_f16_e32 v5, v127                ; encoding: [0x7f,0xbd,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, v127 ; encoding: [0x7f,0xbd,0x0a,0x7e]
 
 v_rndne_f16 v5, s1
-// GFX12: v_rndne_f16_e32 v5, s1                  ; encoding: [0x01,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, s1 ; encoding: [0x01,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, s105
-// GFX12: v_rndne_f16_e32 v5, s105                ; encoding: [0x69,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, s105 ; encoding: [0x69,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, vcc_lo
-// GFX12: v_rndne_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, vcc_hi
-// GFX12: v_rndne_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, ttmp15
-// GFX12: v_rndne_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, m0
-// GFX12: v_rndne_f16_e32 v5, m0                  ; encoding: [0x7d,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, m0 ; encoding: [0x7d,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, exec_lo
-// GFX12: v_rndne_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, exec_lo ; encoding: [0x7e,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, exec_hi
-// GFX12: v_rndne_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, exec_hi ; encoding: [0x7f,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, null
-// GFX12: v_rndne_f16_e32 v5, null                ; encoding: [0x7c,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, null ; encoding: [0x7c,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, -1
-// GFX12: v_rndne_f16_e32 v5, -1                  ; encoding: [0xc1,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, -1 ; encoding: [0xc1,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, 0.5
-// GFX12: v_rndne_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, 0.5 ; encoding: [0xf0,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, src_scc
-// GFX12: v_rndne_f16_e32 v5, src_scc             ; encoding: [0xfd,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, src_scc ; encoding: [0xfd,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v127, 0xfe0b
-// GFX12: v_rndne_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xbc,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rndne_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xbc,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rndne_f32 v5, v1
-// GFX12: v_rndne_f32_e32 v5, v1                  ; encoding: [0x01,0x47,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, v1 ; encoding: [0x01,0x47,0x0a,0x7e]
 
 v_rndne_f32 v5, v255
-// GFX12: v_rndne_f32_e32 v5, v255                ; encoding: [0xff,0x47,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, v255 ; encoding: [0xff,0x47,0x0a,0x7e]
 
 v_rndne_f32 v5, s1
-// GFX12: v_rndne_f32_e32 v5, s1                  ; encoding: [0x01,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, s1 ; encoding: [0x01,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, s105
-// GFX12: v_rndne_f32_e32 v5, s105                ; encoding: [0x69,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, s105 ; encoding: [0x69,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, vcc_lo
-// GFX12: v_rndne_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, vcc_hi
-// GFX12: v_rndne_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, ttmp15
-// GFX12: v_rndne_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, m0
-// GFX12: v_rndne_f32_e32 v5, m0                  ; encoding: [0x7d,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, m0 ; encoding: [0x7d,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, exec_lo
-// GFX12: v_rndne_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, exec_lo ; encoding: [0x7e,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, exec_hi
-// GFX12: v_rndne_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, exec_hi ; encoding: [0x7f,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, null
-// GFX12: v_rndne_f32_e32 v5, null                ; encoding: [0x7c,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, null ; encoding: [0x7c,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, -1
-// GFX12: v_rndne_f32_e32 v5, -1                  ; encoding: [0xc1,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, -1 ; encoding: [0xc1,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, 0.5
-// GFX12: v_rndne_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, 0.5 ; encoding: [0xf0,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, src_scc
-// GFX12: v_rndne_f32_e32 v5, src_scc             ; encoding: [0xfd,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, src_scc ; encoding: [0xfd,0x46,0x0a,0x7e]
 
 v_rndne_f32 v255, 0xaf123456
-// GFX12: v_rndne_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x46,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rndne_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x46,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rndne_f64 v[5:6], v[1:2]
-// GFX12: v_rndne_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x33,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x33,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], v[254:255]
-// GFX12: v_rndne_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x33,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x33,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], s[2:3]
-// GFX12: v_rndne_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], s[104:105]
-// GFX12: v_rndne_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], vcc
-// GFX12: v_rndne_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], ttmp[14:15]
-// GFX12: v_rndne_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], exec
-// GFX12: v_rndne_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], exec ; encoding: [0x7e,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], null
-// GFX12: v_rndne_f64_e32 v[5:6], null            ; encoding: [0x7c,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], null ; encoding: [0x7c,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], -1
-// GFX12: v_rndne_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], 0.5
-// GFX12: v_rndne_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], src_scc
-// GFX12: v_rndne_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[254:255], 0xaf123456
-// GFX12: v_rndne_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rndne_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rsq_f16 v5, v1
-// GFX12: v_rsq_f16_e32 v5, v1                    ; encoding: [0x01,0xad,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, v1 ; encoding: [0x01,0xad,0x0a,0x7e]
 
 v_rsq_f16 v5, v127
-// GFX12: v_rsq_f16_e32 v5, v127                  ; encoding: [0x7f,0xad,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, v127 ; encoding: [0x7f,0xad,0x0a,0x7e]
 
 v_rsq_f16 v5, s1
-// GFX12: v_rsq_f16_e32 v5, s1                    ; encoding: [0x01,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, s1 ; encoding: [0x01,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, s105
-// GFX12: v_rsq_f16_e32 v5, s105                  ; encoding: [0x69,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, s105 ; encoding: [0x69,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, vcc_lo
-// GFX12: v_rsq_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, vcc_hi
-// GFX12: v_rsq_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, ttmp15
-// GFX12: v_rsq_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, m0
-// GFX12: v_rsq_f16_e32 v5, m0                    ; encoding: [0x7d,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, m0 ; encoding: [0x7d,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, exec_lo
-// GFX12: v_rsq_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, exec_lo ; encoding: [0x7e,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, exec_hi
-// GFX12: v_rsq_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, exec_hi ; encoding: [0x7f,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, null
-// GFX12: v_rsq_f16_e32 v5, null                  ; encoding: [0x7c,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, null ; encoding: [0x7c,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, -1
-// GFX12: v_rsq_f16_e32 v5, -1                    ; encoding: [0xc1,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, -1 ; encoding: [0xc1,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, 0.5
-// GFX12: v_rsq_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, 0.5 ; encoding: [0xf0,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, src_scc
-// GFX12: v_rsq_f16_e32 v5, src_scc               ; encoding: [0xfd,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, src_scc ; encoding: [0xfd,0xac,0x0a,0x7e]
 
 v_rsq_f16 v127, 0xfe0b
-// GFX12: v_rsq_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xac,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rsq_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xac,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rsq_f32 v5, v1
-// GFX12: v_rsq_f32_e32 v5, v1                    ; encoding: [0x01,0x5d,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, v1 ; encoding: [0x01,0x5d,0x0a,0x7e]
 
 v_rsq_f32 v5, v255
-// GFX12: v_rsq_f32_e32 v5, v255                  ; encoding: [0xff,0x5d,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, v255 ; encoding: [0xff,0x5d,0x0a,0x7e]
 
 v_rsq_f32 v5, s1
-// GFX12: v_rsq_f32_e32 v5, s1                    ; encoding: [0x01,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, s1 ; encoding: [0x01,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, s105
-// GFX12: v_rsq_f32_e32 v5, s105                  ; encoding: [0x69,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, s105 ; encoding: [0x69,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, vcc_lo
-// GFX12: v_rsq_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, vcc_hi
-// GFX12: v_rsq_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, ttmp15
-// GFX12: v_rsq_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, m0
-// GFX12: v_rsq_f32_e32 v5, m0                    ; encoding: [0x7d,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, m0 ; encoding: [0x7d,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, exec_lo
-// GFX12: v_rsq_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, exec_lo ; encoding: [0x7e,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, exec_hi
-// GFX12: v_rsq_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, exec_hi ; encoding: [0x7f,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, null
-// GFX12: v_rsq_f32_e32 v5, null                  ; encoding: [0x7c,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, null ; encoding: [0x7c,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, -1
-// GFX12: v_rsq_f32_e32 v5, -1                    ; encoding: [0xc1,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, -1 ; encoding: [0xc1,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, 0.5
-// GFX12: v_rsq_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, 0.5 ; encoding: [0xf0,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, src_scc
-// GFX12: v_rsq_f32_e32 v5, src_scc               ; encoding: [0xfd,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, src_scc ; encoding: [0xfd,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v255, 0xaf123456
-// GFX12: v_rsq_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x5c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rsq_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x5c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rsq_f64 v[5:6], v[1:2]
-// GFX12: v_rsq_f64_e32 v[5:6], v[1:2]            ; encoding: [0x01,0x63,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x63,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], v[254:255]
-// GFX12: v_rsq_f64_e32 v[5:6], v[254:255]        ; encoding: [0xfe,0x63,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x63,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], s[2:3]
-// GFX12: v_rsq_f64_e32 v[5:6], s[2:3]            ; encoding: [0x02,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], s[104:105]
-// GFX12: v_rsq_f64_e32 v[5:6], s[104:105]        ; encoding: [0x68,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], vcc
-// GFX12: v_rsq_f64_e32 v[5:6], vcc               ; encoding: [0x6a,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], ttmp[14:15]
-// GFX12: v_rsq_f64_e32 v[5:6], ttmp[14:15]       ; encoding: [0x7a,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], exec
-// GFX12: v_rsq_f64_e32 v[5:6], exec              ; encoding: [0x7e,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], exec ; encoding: [0x7e,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], null
-// GFX12: v_rsq_f64_e32 v[5:6], null              ; encoding: [0x7c,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], null ; encoding: [0x7c,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], -1
-// GFX12: v_rsq_f64_e32 v[5:6], -1                ; encoding: [0xc1,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], 0.5
-// GFX12: v_rsq_f64_e32 v[5:6], 0.5               ; encoding: [0xf0,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], src_scc
-// GFX12: v_rsq_f64_e32 v[5:6], src_scc           ; encoding: [0xfd,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[254:255], 0xaf123456
-// GFX12: v_rsq_f64_e32 v[254:255], 0xaf123456    ; encoding: [0xff,0x62,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rsq_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x62,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sat_pk_u8_i16 v5, v1
-// GFX12: v_sat_pk_u8_i16_e32 v5, v1              ; encoding: [0x01,0xc5,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, v1 ; encoding: [0x01,0xc5,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, v255
-// GFX12: v_sat_pk_u8_i16_e32 v5, v255            ; encoding: [0xff,0xc5,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, v255 ; encoding: [0xff,0xc5,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, s1
-// GFX12: v_sat_pk_u8_i16_e32 v5, s1              ; encoding: [0x01,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, s1 ; encoding: [0x01,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, s105
-// GFX12: v_sat_pk_u8_i16_e32 v5, s105            ; encoding: [0x69,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, s105 ; encoding: [0x69,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, vcc_lo
-// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_lo          ; encoding: [0x6a,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_lo ; encoding: [0x6a,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, vcc_hi
-// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_hi          ; encoding: [0x6b,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_hi ; encoding: [0x6b,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, ttmp15
-// GFX12: v_sat_pk_u8_i16_e32 v5, ttmp15          ; encoding: [0x7b,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, ttmp15 ; encoding: [0x7b,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, m0
-// GFX12: v_sat_pk_u8_i16_e32 v5, m0              ; encoding: [0x7d,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, m0 ; encoding: [0x7d,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, exec_lo
-// GFX12: v_sat_pk_u8_i16_e32 v5, exec_lo         ; encoding: [0x7e,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, exec_lo ; encoding: [0x7e,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, exec_hi
-// GFX12: v_sat_pk_u8_i16_e32 v5, exec_hi         ; encoding: [0x7f,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, exec_hi ; encoding: [0x7f,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, null
-// GFX12: v_sat_pk_u8_i16_e32 v5, null            ; encoding: [0x7c,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, null ; encoding: [0x7c,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, -1
-// GFX12: v_sat_pk_u8_i16_e32 v5, -1              ; encoding: [0xc1,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, -1 ; encoding: [0xc1,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, 0.5
-// GFX12: v_sat_pk_u8_i16_e32 v5, 0.5             ; encoding: [0xf0,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, 0.5 ; encoding: [0xf0,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, src_scc
-// GFX12: v_sat_pk_u8_i16_e32 v5, src_scc         ; encoding: [0xfd,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, src_scc ; encoding: [0xfd,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v127, 0xfe0b
-// GFX12: v_sat_pk_u8_i16_e32 v127, 0xfe0b        ; encoding: [0xff,0xc4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sat_pk_u8_i16_e32 v127, 0xfe0b ; encoding: [0xff,0xc4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sin_f16 v5, v1
-// GFX12: v_sin_f16_e32 v5, v1                    ; encoding: [0x01,0xc1,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, v1 ; encoding: [0x01,0xc1,0x0a,0x7e]
 
 v_sin_f16 v5, v127
-// GFX12: v_sin_f16_e32 v5, v127                  ; encoding: [0x7f,0xc1,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, v127 ; encoding: [0x7f,0xc1,0x0a,0x7e]
 
 v_sin_f16 v5, s1
-// GFX12: v_sin_f16_e32 v5, s1                    ; encoding: [0x01,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, s1 ; encoding: [0x01,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, s105
-// GFX12: v_sin_f16_e32 v5, s105                  ; encoding: [0x69,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, s105 ; encoding: [0x69,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, vcc_lo
-// GFX12: v_sin_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, vcc_hi
-// GFX12: v_sin_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, ttmp15
-// GFX12: v_sin_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, m0
-// GFX12: v_sin_f16_e32 v5, m0                    ; encoding: [0x7d,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, m0 ; encoding: [0x7d,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, exec_lo
-// GFX12: v_sin_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, exec_lo ; encoding: [0x7e,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, exec_hi
-// GFX12: v_sin_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, exec_hi ; encoding: [0x7f,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, null
-// GFX12: v_sin_f16_e32 v5, null                  ; encoding: [0x7c,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, null ; encoding: [0x7c,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, -1
-// GFX12: v_sin_f16_e32 v5, -1                    ; encoding: [0xc1,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, -1 ; encoding: [0xc1,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, 0.5
-// GFX12: v_sin_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, 0.5 ; encoding: [0xf0,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, src_scc
-// GFX12: v_sin_f16_e32 v5, src_scc               ; encoding: [0xfd,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, src_scc ; encoding: [0xfd,0xc0,0x0a,0x7e]
 
 v_sin_f16 v127, 0xfe0b
-// GFX12: v_sin_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xc0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sin_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xc0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sin_f32 v5, v1
-// GFX12: v_sin_f32_e32 v5, v1                    ; encoding: [0x01,0x6b,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, v1 ; encoding: [0x01,0x6b,0x0a,0x7e]
 
 v_sin_f32 v5, v255
-// GFX12: v_sin_f32_e32 v5, v255                  ; encoding: [0xff,0x6b,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, v255 ; encoding: [0xff,0x6b,0x0a,0x7e]
 
 v_sin_f32 v5, s1
-// GFX12: v_sin_f32_e32 v5, s1                    ; encoding: [0x01,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, s1 ; encoding: [0x01,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, s105
-// GFX12: v_sin_f32_e32 v5, s105                  ; encoding: [0x69,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, s105 ; encoding: [0x69,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, vcc_lo
-// GFX12: v_sin_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, vcc_hi
-// GFX12: v_sin_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, ttmp15
-// GFX12: v_sin_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, m0
-// GFX12: v_sin_f32_e32 v5, m0                    ; encoding: [0x7d,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, m0 ; encoding: [0x7d,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, exec_lo
-// GFX12: v_sin_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, exec_lo ; encoding: [0x7e,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, exec_hi
-// GFX12: v_sin_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, exec_hi ; encoding: [0x7f,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, null
-// GFX12: v_sin_f32_e32 v5, null                  ; encoding: [0x7c,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, null ; encoding: [0x7c,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, -1
-// GFX12: v_sin_f32_e32 v5, -1                    ; encoding: [0xc1,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, -1 ; encoding: [0xc1,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, 0.5
-// GFX12: v_sin_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, 0.5 ; encoding: [0xf0,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, src_scc
-// GFX12: v_sin_f32_e32 v5, src_scc               ; encoding: [0xfd,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, src_scc ; encoding: [0xfd,0x6a,0x0a,0x7e]
 
 v_sin_f32 v255, 0xaf123456
-// GFX12: v_sin_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sin_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sqrt_f16 v5, v1
-// GFX12: v_sqrt_f16_e32 v5, v1                   ; encoding: [0x01,0xab,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, v1 ; encoding: [0x01,0xab,0x0a,0x7e]
 
 v_sqrt_f16 v5, v127
-// GFX12: v_sqrt_f16_e32 v5, v127                 ; encoding: [0x7f,0xab,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, v127 ; encoding: [0x7f,0xab,0x0a,0x7e]
 
 v_sqrt_f16 v5, s1
-// GFX12: v_sqrt_f16_e32 v5, s1                   ; encoding: [0x01,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, s1 ; encoding: [0x01,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, s105
-// GFX12: v_sqrt_f16_e32 v5, s105                 ; encoding: [0x69,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, s105 ; encoding: [0x69,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, vcc_lo
-// GFX12: v_sqrt_f16_e32 v5, vcc_lo               ; encoding: [0x6a,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, vcc_hi
-// GFX12: v_sqrt_f16_e32 v5, vcc_hi               ; encoding: [0x6b,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, ttmp15
-// GFX12: v_sqrt_f16_e32 v5, ttmp15               ; encoding: [0x7b,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, m0
-// GFX12: v_sqrt_f16_e32 v5, m0                   ; encoding: [0x7d,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, m0 ; encoding: [0x7d,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, exec_lo
-// GFX12: v_sqrt_f16_e32 v5, exec_lo              ; encoding: [0x7e,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, exec_lo ; encoding: [0x7e,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, exec_hi
-// GFX12: v_sqrt_f16_e32 v5, exec_hi              ; encoding: [0x7f,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, exec_hi ; encoding: [0x7f,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, null
-// GFX12: v_sqrt_f16_e32 v5, null                 ; encoding: [0x7c,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, null ; encoding: [0x7c,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, -1
-// GFX12: v_sqrt_f16_e32 v5, -1                   ; encoding: [0xc1,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, -1 ; encoding: [0xc1,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, 0.5
-// GFX12: v_sqrt_f16_e32 v5, 0.5                  ; encoding: [0xf0,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, 0.5 ; encoding: [0xf0,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, src_scc
-// GFX12: v_sqrt_f16_e32 v5, src_scc              ; encoding: [0xfd,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, src_scc ; encoding: [0xfd,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v127, 0xfe0b
-// GFX12: v_sqrt_f16_e32 v127, 0xfe0b             ; encoding: [0xff,0xaa,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sqrt_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xaa,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sqrt_f32 v5, v1
-// GFX12: v_sqrt_f32_e32 v5, v1                   ; encoding: [0x01,0x67,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, v1 ; encoding: [0x01,0x67,0x0a,0x7e]
 
 v_sqrt_f32 v5, v255
-// GFX12: v_sqrt_f32_e32 v5, v255                 ; encoding: [0xff,0x67,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, v255 ; encoding: [0xff,0x67,0x0a,0x7e]
 
 v_sqrt_f32 v5, s1
-// GFX12: v_sqrt_f32_e32 v5, s1                   ; encoding: [0x01,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, s1 ; encoding: [0x01,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, s105
-// GFX12: v_sqrt_f32_e32 v5, s105                 ; encoding: [0x69,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, s105 ; encoding: [0x69,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, vcc_lo
-// GFX12: v_sqrt_f32_e32 v5, vcc_lo               ; encoding: [0x6a,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, vcc_hi
-// GFX12: v_sqrt_f32_e32 v5, vcc_hi               ; encoding: [0x6b,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, ttmp15
-// GFX12: v_sqrt_f32_e32 v5, ttmp15               ; encoding: [0x7b,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, m0
-// GFX12: v_sqrt_f32_e32 v5, m0                   ; encoding: [0x7d,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, m0 ; encoding: [0x7d,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, exec_lo
-// GFX12: v_sqrt_f32_e32 v5, exec_lo              ; encoding: [0x7e,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, exec_lo ; encoding: [0x7e,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, exec_hi
-// GFX12: v_sqrt_f32_e32 v5, exec_hi              ; encoding: [0x7f,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, exec_hi ; encoding: [0x7f,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, null
-// GFX12: v_sqrt_f32_e32 v5, null                 ; encoding: [0x7c,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, null ; encoding: [0x7c,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, -1
-// GFX12: v_sqrt_f32_e32 v5, -1                   ; encoding: [0xc1,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, -1 ; encoding: [0xc1,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, 0.5
-// GFX12: v_sqrt_f32_e32 v5, 0.5                  ; encoding: [0xf0,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, 0.5 ; encoding: [0xf0,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, src_scc
-// GFX12: v_sqrt_f32_e32 v5, src_scc              ; encoding: [0xfd,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, src_scc ; encoding: [0xfd,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v255, 0xaf123456
-// GFX12: v_sqrt_f32_e32 v255, 0xaf123456         ; encoding: [0xff,0x66,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sqrt_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x66,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sqrt_f64 v[5:6], v[1:2]
-// GFX12: v_sqrt_f64_e32 v[5:6], v[1:2]           ; encoding: [0x01,0x69,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x69,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], v[254:255]
-// GFX12: v_sqrt_f64_e32 v[5:6], v[254:255]       ; encoding: [0xfe,0x69,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x69,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], s[2:3]
-// GFX12: v_sqrt_f64_e32 v[5:6], s[2:3]           ; encoding: [0x02,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], s[104:105]
-// GFX12: v_sqrt_f64_e32 v[5:6], s[104:105]       ; encoding: [0x68,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], vcc
-// GFX12: v_sqrt_f64_e32 v[5:6], vcc              ; encoding: [0x6a,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], ttmp[14:15]
-// GFX12: v_sqrt_f64_e32 v[5:6], ttmp[14:15]      ; encoding: [0x7a,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], exec
-// GFX12: v_sqrt_f64_e32 v[5:6], exec             ; encoding: [0x7e,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], exec ; encoding: [0x7e,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], null
-// GFX12: v_sqrt_f64_e32 v[5:6], null             ; encoding: [0x7c,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], null ; encoding: [0x7c,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], -1
-// GFX12: v_sqrt_f64_e32 v[5:6], -1               ; encoding: [0xc1,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], 0.5
-// GFX12: v_sqrt_f64_e32 v[5:6], 0.5              ; encoding: [0xf0,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], src_scc
-// GFX12: v_sqrt_f64_e32 v[5:6], src_scc          ; encoding: [0xfd,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[254:255], 0xaf123456
-// GFX12: v_sqrt_f64_e32 v[254:255], 0xaf123456   ; encoding: [0xff,0x68,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sqrt_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x68,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_swap_b16 v5.l, v1.h
-// GFX12: v_swap_b16  v5.l, v1.h                  ; encoding: [0x81,0xcd,0x0a,0x7e]
+// GFX12: v_swap_b16  v5.l, v1.h ; encoding: [0x81,0xcd,0x0a,0x7e]
 
 v_swap_b16 v5.h, v1.l
-// GFX12: v_swap_b16  v5.h, v1.l                  ; encoding: [0x01,0xcd,0x0a,0x7f]
+// GFX12: v_swap_b16  v5.h, v1.l ; encoding: [0x01,0xcd,0x0a,0x7f]
 
 v_swap_b16 v127.l, v127.l
-// GFX12: v_swap_b16  v127.l, v127.l              ; encoding: [0x7f,0xcd,0xfe,0x7e]
+// GFX12: v_swap_b16  v127.l, v127.l ; encoding: [0x7f,0xcd,0xfe,0x7e]
 
 v_swap_b32 v5, v1
-// GFX12: v_swap_b32 v5, v1                       ; encoding: [0x01,0xcb,0x0a,0x7e]
+// GFX12: v_swap_b32 v5, v1 ; encoding: [0x01,0xcb,0x0a,0x7e]
 
 v_swap_b32 v255, v255
-// GFX12: v_swap_b32 v255, v255                   ; encoding: [0xff,0xcb,0xfe,0x7f]
+// GFX12: v_swap_b32 v255, v255 ; encoding: [0xff,0xcb,0xfe,0x7f]
 
 v_swaprel_b32 v5, v1
-// GFX12: v_swaprel_b32 v5, v1                    ; encoding: [0x01,0xd1,0x0a,0x7e]
+// GFX12: v_swaprel_b32 v5, v1 ; encoding: [0x01,0xd1,0x0a,0x7e]
 
 v_swaprel_b32 v255, v255
-// GFX12: v_swaprel_b32 v255, v255                ; encoding: [0xff,0xd1,0xfe,0x7f]
+// GFX12: v_swaprel_b32 v255, v255 ; encoding: [0xff,0xd1,0xfe,0x7f]
 
 v_trunc_f16 v5, v1
-// GFX12: v_trunc_f16_e32 v5, v1                  ; encoding: [0x01,0xbb,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, v1 ; encoding: [0x01,0xbb,0x0a,0x7e]
 
 v_trunc_f16 v5, v127
-// GFX12: v_trunc_f16_e32 v5, v127                ; encoding: [0x7f,0xbb,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, v127 ; encoding: [0x7f,0xbb,0x0a,0x7e]
 
 v_trunc_f16 v5, s1
-// GFX12: v_trunc_f16_e32 v5, s1                  ; encoding: [0x01,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, s1 ; encoding: [0x01,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, s105
-// GFX12: v_trunc_f16_e32 v5, s105                ; encoding: [0x69,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, s105 ; encoding: [0x69,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, vcc_lo
-// GFX12: v_trunc_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, vcc_hi
-// GFX12: v_trunc_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, ttmp15
-// GFX12: v_trunc_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, m0
-// GFX12: v_trunc_f16_e32 v5, m0                  ; encoding: [0x7d,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, m0 ; encoding: [0x7d,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, exec_lo
-// GFX12: v_trunc_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, exec_lo ; encoding: [0x7e,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, exec_hi
-// GFX12: v_trunc_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, exec_hi ; encoding: [0x7f,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, null
-// GFX12: v_trunc_f16_e32 v5, null                ; encoding: [0x7c,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, null ; encoding: [0x7c,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, -1
-// GFX12: v_trunc_f16_e32 v5, -1                  ; encoding: [0xc1,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, -1 ; encoding: [0xc1,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, 0.5
-// GFX12: v_trunc_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, 0.5 ; encoding: [0xf0,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, src_scc
-// GFX12: v_trunc_f16_e32 v5, src_scc             ; encoding: [0xfd,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, src_scc ; encoding: [0xfd,0xba,0x0a,0x7e]
 
 v_trunc_f16 v127, 0xfe0b
-// GFX12: v_trunc_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xba,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_trunc_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xba,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_trunc_f32 v5, v1
-// GFX12: v_trunc_f32_e32 v5, v1                  ; encoding: [0x01,0x43,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, v1 ; encoding: [0x01,0x43,0x0a,0x7e]
 
 v_trunc_f32 v5, v255
-// GFX12: v_trunc_f32_e32 v5, v255                ; encoding: [0xff,0x43,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, v255 ; encoding: [0xff,0x43,0x0a,0x7e]
 
 v_trunc_f32 v5, s1
-// GFX12: v_trunc_f32_e32 v5, s1                  ; encoding: [0x01,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, s1 ; encoding: [0x01,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, s105
-// GFX12: v_trunc_f32_e32 v5, s105                ; encoding: [0x69,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, s105 ; encoding: [0x69,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, vcc_lo
-// GFX12: v_trunc_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, vcc_hi
-// GFX12: v_trunc_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, ttmp15
-// GFX12: v_trunc_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, m0
-// GFX12: v_trunc_f32_e32 v5, m0                  ; encoding: [0x7d,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, m0 ; encoding: [0x7d,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, exec_lo
-// GFX12: v_trunc_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, exec_lo ; encoding: [0x7e,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, exec_hi
-// GFX12: v_trunc_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, exec_hi ; encoding: [0x7f,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, null
-// GFX12: v_trunc_f32_e32 v5, null                ; encoding: [0x7c,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, null ; encoding: [0x7c,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, -1
-// GFX12: v_trunc_f32_e32 v5, -1                  ; encoding: [0xc1,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, -1 ; encoding: [0xc1,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, 0.5
-// GFX12: v_trunc_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, 0.5 ; encoding: [0xf0,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, src_scc
-// GFX12: v_trunc_f32_e32 v5, src_scc             ; encoding: [0xfd,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, src_scc ; encoding: [0xfd,0x42,0x0a,0x7e]
 
 v_trunc_f32 v255, 0xaf123456
-// GFX12: v_trunc_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x42,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_trunc_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x42,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_trunc_f64 v[5:6], v[1:2]
-// GFX12: v_trunc_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x2f,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x2f,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], v[254:255]
-// GFX12: v_trunc_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x2f,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x2f,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], s[2:3]
-// GFX12: v_trunc_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], s[104:105]
-// GFX12: v_trunc_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], vcc
-// GFX12: v_trunc_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], ttmp[14:15]
-// GFX12: v_trunc_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], exec
-// GFX12: v_trunc_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], exec ; encoding: [0x7e,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], null
-// GFX12: v_trunc_f64_e32 v[5:6], null            ; encoding: [0x7c,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], null ; encoding: [0x7c,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], -1
-// GFX12: v_trunc_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], 0.5
-// GFX12: v_trunc_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], src_scc
-// GFX12: v_trunc_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[254:255], 0xaf123456
-// GFX12: v_trunc_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_trunc_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
index af4cb1a008f9d92..bec717e4137df2b 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
@@ -106,44 +106,44 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1)>>0
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&32)>>5
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
-// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset ((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
 // ASM-NEXT: .amdhsa_reserve_flat_scratch defined_boolean
 // ASM-NEXT: .amdhsa_reserve_xnack_mask 1
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&67108864)>>26
-// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&536870912)>>29
-// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&1073741824)>>30
-// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2147483648)>>31
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2147483648)>>31
 // ASM-NEXT: .amdhsa_shared_vgpr_count 0
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
index b6e4ddde3d7f9e2..85a7ad05b00f484 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
@@ -106,40 +106,40 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
-// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_enable_private_segment ((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&67108864)>>26
-// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&536870912)>>29
-// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&1073741824)>>30
-// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2147483648)>>31
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2147483648)>>31
 // ASM-NEXT: .amdhsa_shared_vgpr_count 0
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
index a80000dc44dac0d..51d0fb30b320c52 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
@@ -108,38 +108,38 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
-// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_enable_private_segment ((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&67108864)>>26
-// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&536870912)>>29
-// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&1073741824)>>30
-// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2147483648)>>31
-// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&786432)>>18
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2147483648)>>31
+// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2097152)>>21
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
index 7ab2e2b28a0e6ec..485f48c695c4dea 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
@@ -98,37 +98,37 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset ((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
 // ASM-NEXT: .amdhsa_reserve_flat_scratch defined_boolean
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_float_round_mode_32 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
index caccde7ba0e3b8e..0d2e066113ee861 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
@@ -99,38 +99,38 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset ((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
 // ASM-NEXT: .amdhsa_reserve_flat_scratch defined_boolean
 // ASM-NEXT: .amdhsa_reserve_xnack_mask 1
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_float_round_mode_32 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
index 216ae4c42a3d9a9..88b5e23a6f2c5f1 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
@@ -70,7 +70,7 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size 0
 // ASM-NEXT: .amdhsa_private_segment_fixed_size 0
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
@@ -80,33 +80,33 @@ expr_defined:
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
-// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset ((128|defined_boolean)&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((128|defined_boolean)&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((128|defined_boolean)&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((128|defined_boolean)&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((128|defined_boolean)&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_boolean+1
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_boolean+2
-// ASM-NEXT: .amdhsa_accum_offset (((((((0&(~65536))|(defined_boolean<<16))&(~63))|(((4/4)-1)<<0))&63)>>0)+1)*4
+// ASM-NEXT: .amdhsa_accum_offset 4
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
 // ASM-NEXT: .amdhsa_reserve_flat_scratch defined_boolean
 // ASM-NEXT: .amdhsa_reserve_xnack_mask 1
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&67108864)>>26
-// ASM-NEXT: .amdhsa_tg_split (((((0&(~65536))|(defined_boolean<<16))&(~63))|(((4/4)-1)<<0))&65536)>>16 
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_float_round_mode_32 ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_dx10_clamp ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&67108864)>>26
+// ASM-NEXT: .amdhsa_tg_split (((defined_boolean<<16)&(~63))&65536)>>16
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((128|defined_boolean)&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((128|defined_boolean)&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((128|defined_boolean)&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((128|defined_boolean)&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((128|defined_boolean)&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((128|defined_boolean)&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((128|defined_boolean)&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_boolean, 1
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.5a-ssbs.txt b/llvm/test/MC/Disassembler/AArch64/armv8.5a-ssbs.txt
index 7698751c88076b0..84d4fa6accccf48 100644
--- a/llvm/test/MC/Disassembler/AArch64/armv8.5a-ssbs.txt
+++ b/llvm/test/MC/Disassembler/AArch64/armv8.5a-ssbs.txt
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple=aarch64 -mattr=+ssbs  -disassemble < %s | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+v8.5a -disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple=aarch64 -mattr=+v8.5a -disassemble < %s | FileCheck %s --check-prefix=NOSPECID
 # RUN: llvm-mc -triple=aarch64 -mcpu=cortex-a76 -disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple=aarch64 -mcpu=cortex-a76ae -disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple=aarch64 -mattr=+v8r -disassemble < %s | FileCheck %s --check-prefix=NOSPECID
diff --git a/llvm/test/MC/Disassembler/AArch64/basic-a64-instructions.txt b/llvm/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
index c76bb0b9020967b..f46301e8c1c15b7 100644
--- a/llvm/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
+++ b/llvm/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
@@ -2,7 +2,7 @@
 # RUN: llvm-mc -triple=arm64 -mattr=+v8a,+fp-armv8 -disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple=arm64 -mattr=+v8a,+fp-armv8,+fullfp16 -disassemble < %s | FileCheck %s --check-prefix=CHECK --check-prefix=FP16
 # RUN: llvm-mc -triple=arm64 -mattr=+v8.2a -disassemble < %s | FileCheck %s --check-prefix=CHECK-V82
-# RUN: llvm-mc -triple=arm64 -mattr=+v8.3a -disassemble < %s | FileCheck %s --check-prefix=CHECK-V83
+# RUN: llvm-mc -triple=arm64 -mattr=+ccidx -disassemble < %s | FileCheck %s --check-prefix=CHECK-V83
 
 #------------------------------------------------------------------------------
 # Add/sub (immediate)
diff --git a/llvm/test/MC/ELF/relax-branch.s b/llvm/test/MC/ELF/relax-branch.s
new file mode 100644
index 000000000000000..5abf8355f987627
--- /dev/null
+++ b/llvm/test/MC/ELF/relax-branch.s
@@ -0,0 +1,398 @@
+## A larger branch relaxation test that requires multiple relaxation steps.
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: llvm-objdump -d %t | FileCheck %s
+
+## Regression test that we use jump short instead of jump near
+## (see the reverted be5a845e4c29aadb513ae6e5e2879dccf37efdbb).
+# CHECK:      4a9: eb 00                         jmp     0x4ab
+# CHECK:      5a3: eb 00                         jmp     0x5a5
+# CHECK-NEXT: 5a5: eb 00                         jmp     0x5a7
+# CHECK-NEXT: 5a7: eb 00                         jmp     0x5a9
+# CHECK-NEXT: 5a9: eb 00                         jmp     0x5ab
+# CHECK-NEXT: 5ab: e9 fb fe ff ff                jmp     0x4ab
+# CHECK-NEXT: 5b0: eb 00                         jmp     0x5b2
+
+	pushq	%rbp
+	movq	%rsp, %rbp
+	movq	%rcx, (%rbp)
+	movq	%r8, (%rbp)
+	movl	$0, (%rbp)
+	movl	$0, (%rbp)
+	movl	$0, (%rbp)
+	cmpq	$0, (%rbp)
+	je	.LBB0_2
+	movl	$0, (%rbp)
+	jmp	.LBB0_30
+.LBB0_2:
+	jmp	.LBB0_3
+.LBB0_3:
+	movq	(%rbp), %rax
+	movsbl	(%rax), %ecx
+	xorl	%eax, %eax
+	cmpl	%ecx, %eax
+	jne	.LBB0_29
+	jmp	.LBB0_5
+.LBB0_5:
+	movb	$0, %al
+	callq	sqlite3_stepPLT
+	movl	%eax, (%rbp)
+	cmpq	$0, (%rbp)
+	je	.LBB0_23
+	movl	$100, %eax
+	cmpl	(%rbp), %eax
+	sete	%al
+	andb	$1, %al
+	movzbl	%al, %eax
+	cmpl	(%rbp), %eax
+	jne	.LBB0_23
+	cmpl	$0, (%rbp)
+	je	.LBB0_23
+	movq	(%rbp), %rax
+	movl	(%rax), %eax
+	andl	$256, %eax
+	cmpl	$0, %eax
+	je	.LBB0_23
+	cmpl	$0, (%rbp)
+	je	.LBB0_22
+	movl	$0, (%rbp)
+.LBB0_11:
+	cmpl	$0, (%rbp)
+	je	.LBB0_13
+	movl	(%rbp), %esi
+	xorl	%edi, %edi
+	movb	$0, %al
+	callq	sqlite3_free_tablePLT
+	jmp	.LBB0_11
+.LBB0_13:
+	movq	(%rbp), %rax
+	movslq	(%rbp), %rcx
+	cmpb	$0, (%rax,%rcx)
+	je	.LBB0_15
+	movq	(%rbp), %rax
+	movb	$1, 8(%rax)
+.LBB0_15:
+	movq	(%rbp), %rax
+	movslq	(%rbp), %rcx
+	addq	%rcx, %rax
+	movq	%rax, (%rip)
+	movl	$0, (%rbp)
+.LBB0_16:
+	movl	(%rbp), %eax
+	cmpl	(%rbp), %eax
+	jge	.LBB0_21
+	movl	$0, (%rbp)
+	movl	$0, (%rbp)
+	movq	(%rbp), %rdi
+	movb	$0, %al
+	callq	sqlite3_free_tablePLT
+	movq	(%rip), %rax
+	movslq	(%rbp), %rcx
+	movl	$5, %edx
+	cmpq	%rdx, (%rax,%rcx,)
+	je	.LBB0_19
+	movq	(%rbp), %rax
+	movb	$1, 8(%rax)
+	jmp	.LBB0_30
+.LBB0_19:
+	jmp	.LBB0_20
+.LBB0_20:
+	movl	(%rbp), %eax
+	addl	$1, %eax
+	movl	%eax, (%rbp)
+	jmp	.LBB0_16
+.LBB0_21:
+	jmp	.LBB0_22
+.LBB0_22:
+	jmp	.LBB0_23
+.LBB0_23:
+	cmpl	$0, (%rbp)
+	je	.LBB0_27
+	movb	$0, %al
+	callq	sqlite3_stepPLT
+	movl	%eax, (%rbp)
+	movl	$0, (%rbp)
+	cmpl	$0, (%rbp)
+	je	.LBB0_26
+	jmp	.LBB0_28
+.LBB0_26:
+	jmp	.LBB0_27
+.LBB0_27:
+	jmp	.LBB0_5
+.LBB0_28:
+	jmp	.LBB0_3
+.LBB0_29:
+	jmp	.LBB0_30
+.LBB0_30:
+	movl	(%rbp), %eax
+	addq	$96, %rsp
+	popq	%rbp
+	retq
+	.p2align	4, 0x90
+	movq	%rsp, %rbp
+	subq	$48, %rsp
+	movq	%rdi, (%rbp)
+	movl	%esi, (%rbp)
+	movl	%edx, (%rbp)
+	cmpq	$0, sqlite3_get_table_res_00
+	movq	(%rbp), %rax
+	jmp	.LBB1_8
+	movq	$0, (%rbp)
+	movq	(%rbp), %rdi
+	leaq	sqlite3_get_table_zSql0, %rsi
+	leaq	sqlite3_get_table_cb0, %rdx
+	leaq	sqlite3_get_table_res0, %rcx
+	leaq	(%rip), %r8
+	callq	sqlite3_exec
+	movl	%eax, (%rip)
+	movslq	(%rbp), %rax
+	movq	%rax, (%rbp)
+	movl	(%rip), %eax
+	andl	$5, %eax
+	cmpl	$0, %eax
+	je	.LBB1_4
+	movl	(%rip), %ecx
+	movq	(%rbp), %rax
+	movl	%ecx, (%rax)
+	movl	(rip), %eax
+	movl	%eax, (%rbp)
+	jmp	.LBB1_8
+.LBB1_4:
+	movsbl	(%rip), %edi
+	movb	$0, %al
+	callq	sqlite3_free_tablePLT
+	cmpl	$0, (%rip)
+	je	.LBB1_6
+	movq	(%rip), %rax
+	movsbl	(%rax), %edi
+	movb	$0, %al
+	callq	sqlite3_free_tablePLT
+.LBB1_6:
+	movq	(rip), %rdi
+	movslq	(%rbp), %rsi
+	shlq	$0, %rsi
+	addq	$1, %rsi
+	movb	$0, %al
+	callq	sqlite3_mallocPLT
+	movq	%rax, (%rbp)
+	cmpq	$0, (%rbp)
+	je	.LBB1_8
+	movl	$1, (%rbp)
+.LBB1_8:
+	movl	(%rbp), %eax
+	addq	$48, %rsp
+	.p2align	4, 0x90
+	movq	(%rbp), %rax
+	movl	28(%rax), %eax
+	movq	(%rbp), %rcx
+	addl	16(%rcx), %eax
+	cmpl	$0, %eax
+	movq	(%rbp), %rax
+	movl	16(%rax), %ecx
+	addl	(%rbp), %ecx
+	addl	$1, %ecx
+	movq	(%rbp), %rax
+	movl	%ecx, 16(%rax)
+	jmp	.LBB2_14
+	movq	(%rbp), %rax
+	cmpl	$0, 20(%rax)
+	je	.LBB2_10
+	movl	$0, (%rbp)
+	movq	(%rbp), %rax
+	movl	$0, 24(%rax)
+.LBB2_4:
+	movl	(%rbp), %eax
+	cmpl	(%rbp), %eax
+	jge	.LBB2_7
+	movq	(%rbp), %rax
+	movslq	(%rbp), %rcx
+	movsbl	(%rax,%rcx), %esi
+	leaq	.str, %rdi
+	movb	$0, %al
+	callq	sqlite3_mprintf@PLT
+	movq	%rax, -64(%rbp)
+	movl	-56(%rbp), %eax
+	addl	$1, %eax
+	movl	%eax, -56(%rbp)
+	jmp	.LBB2_4
+.LBB2_7:
+	cmpq	$0, -64(%rbp)
+	je	.LBB2_9
+	jmp	.LBB2_9
+.LBB2_9:
+	movq	-64(%rbp), %rdx
+	movq	-48(%rbp), %rax
+	movq	(%rax), %rax
+	movq	-48(%rbp), %rsi
+	movl	28(%rsi), %ecx
+	movl	%ecx, %edi
+	addl	$1, %edi
+	movl	%edi, 28(%rsi)
+	movslq	%ecx, %rcx
+	movq	%rdx, (%rax,%rcx,8)
+	jmp	.LBB2_13
+.LBB2_10:
+	movq	-48(%rbp), %rax
+	movslq	-20(%rbp), %rcx
+	cmpq	%rcx, %rax
+	je	.LBB2_12
+	movq	-48(%rbp), %rax
+	movq	8(%rax), %rdi
+	movb	$0, %al
+	callq	sqlite3_free_table@PLT
+	leaq	.L.str, %rdi
+	movb	$0, %al
+	callq	sqlite3_mprintf@PLT
+	movq	%rax, %rcx
+	movq	-48(%rbp), %rax
+	movq	%rcx, 8(%rax)
+	movq	-48(%rbp), %rax
+	movl	$1, 32(%rax)
+	movl	$1, -4(%rbp)
+	jmp	.LBB2_15
+.LBB2_12:
+	jmp	.LBB2_13
+.LBB2_13:
+	movl	$0, -4(%rbp)
+	jmp	.LBB2_15
+.LBB2_14:
+.LBB2_15:
+	movl	-4(%rbp), %eax
+	addq	$64, %rsp
+	popq	%rbp
+	.p2align	4, 0x90
+sqlite3_complete:
+	je	.LBB3_54
+	je	.LBB3_25
+	je	.LBB3_25
+	je	.LBB3_25
+	jmp	.LBB3_28
+	movl	$42, %eax
+	cmpq	%rax, -160
+	je	.LBB3_5
+	jmp	.LBB3_53
+.LBB3_5:
+	jmp	.LBB3_6
+.LBB3_6:
+	movq	-160, %rax
+	movsbl	0, %ecx
+	xorl	%eax, %eax
+	movb	$1, %al
+	movl	$42, %ecx
+	cmpq	%rcx, -160
+	movb	%al, -340
+	jne	.LBB3_9
+	movq	-160, %rax
+	movsbl	10, %eax
+	cmpl	$0, %eax
+	setne	%al
+	movb	%al, -340
+.LBB3_9:
+	movb	-340, %al
+	movb	%al, -330
+	movb	-330, %al
+	testb	$1, %al
+	jne	.LBB3_11
+	jmp	.LBB3_14
+.LBB3_11:
+	movq	-160, %rax
+	movsbl	0, %eax
+	cmpl	$0, %eax
+	jne	.LBB3_13
+	movl	$0, -40
+	jmp	.LBB3_54
+.LBB3_13:
+	jmp	.LBB3_6
+.LBB3_14:
+	jmp	.LBB3_53
+	movq	-160, %rax
+	movsbl	10, %eax
+	cmpl	$45, %eax
+	je	.LBB3_17
+	movb	$2, -170
+	jmp	.LBB3_53
+.LBB3_17:
+	jmp	.LBB3_18
+.LBB3_18:
+	movq	-160, %rax
+	movsbl	0, %ecx
+	xorl	%eax, %eax
+	cmpl	$0, %ecx
+	movb	%al, -350
+	jmp	.LBB3_53
+.LBB3_25:
+	movl	$0, -40
+	jmp	.LBB3_54
+	jmp	.LBB3_53
+.LBB3_28:
+	je	.LBB3_52
+	jmp	.LBB3_52
+	jmp	.LBB3_31
+.LBB3_31:
+	movq	-160, %rax
+	movslq	-280, %rcx
+	movsbl	0, %ecx
+	movb	-360, %al
+	andb	$1, %cl
+	movzbl	%cl, %ecx
+	movl	%ecx, -240
+	testb	$1, %al
+	jne	.LBB3_36
+	jmp	.LBB3_51
+.LBB3_36:
+	movq	-160, %rax
+	movsbl	0, %eax
+	orl	$32, %eax
+	cmpl	$0, -280
+	je	.LBB3_40
+	movq	-160, %rdi
+	leaq	.L.str, %rsi
+	movl	$7, %edx
+	movb	$0, %al
+	callq	sqlite3StrNICmp@PLT
+	cmpl	$0, %eax
+	je	.LBB3_40
+	jmp	.LBB3_48
+.LBB3_40:
+	cmpl	$0, -280
+	je	.LBB3_43
+	movq	-160, %rdi
+	leaq	.L.str, %rsi
+	movl	$4, %edx
+	movb	$0, %al
+	callq	sqlite3StrNICmp@PLT
+	cmpl	$0, %eax
+	je	.LBB3_43
+	movb	$5, -170
+	jmp	.LBB3_47
+.LBB3_43:
+	cmpl	$0, -280
+	je	.LBB3_46
+	movq	-160, %rdi
+	leaq	.L.str, %rsi
+	movl	$9, %edx
+	movb	$0, %al
+	callq	sqlite3StrNICmp@PLT
+	cmpl	$0, %eax
+	je	.LBB3_46
+	movl	-280, %ecx
+	subl	$1, %ecx
+	movq	-160, %rax
+	movslq	%ecx, %rcx
+	addq	%rcx, %rax
+	movq	%rax, -160
+.LBB3_46:
+	jmp	.LBB3_47
+.LBB3_47:
+	jmp	.LBB3_48
+.LBB3_48:
+	jmp	.LBB3_49
+.LBB3_49:
+	jmp	.LBB3_50
+.LBB3_50:
+	jmp	.LBB3_31
+.LBB3_51:
+	jmp	.LBB3_52
+.LBB3_52:
+.LBB3_53:
+.LBB3_54:
+.L.str:
diff --git a/llvm/test/MC/RISCV/machine-csr-names-invalid.s b/llvm/test/MC/RISCV/machine-csr-names-invalid.s
index 9afb6d7744a8df1..526c41773484ca0 100644
--- a/llvm/test/MC/RISCV/machine-csr-names-invalid.s
+++ b/llvm/test/MC/RISCV/machine-csr-names-invalid.s
@@ -3,38 +3,38 @@
 
 # These machine mode CSR register names are RV32 only.
 
-csrrs t1, pmpcfg1, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg3, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, pmpcfg1, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg1' is RV32 only
+csrrs t1, pmpcfg3, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg3' is RV32 only
 
-csrrs t1, mcycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, minstreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, mcycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mcycleh' is RV32 only
+csrrs t1, minstreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'minstreth' is RV32 only
 
-csrrs t1, mhpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, mhpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter3h' is RV32 only
+csrrs t1, mhpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter4h' is RV32 only
+csrrs t1, mhpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter5h' is RV32 only
+csrrs t1, mhpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter6h' is RV32 only
+csrrs t1, mhpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter7h' is RV32 only
+csrrs t1, mhpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter8h' is RV32 only
+csrrs t1, mhpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter9h' is RV32 only
+csrrs t1, mhpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter10h' is RV32 only
+csrrs t1, mhpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter11h' is RV32 only
+csrrs t1, mhpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter12h' is RV32 only
+csrrs t1, mhpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter13h' is RV32 only
+csrrs t1, mhpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter14h' is RV32 only
+csrrs t1, mhpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter15h' is RV32 only
+csrrs t1, mhpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter16h' is RV32 only
+csrrs t1, mhpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter17h' is RV32 only
+csrrs t1, mhpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter18h' is RV32 only
+csrrs t1, mhpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter19h' is RV32 only
+csrrs t1, mhpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter20h' is RV32 only
+csrrs t1, mhpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter21h' is RV32 only
+csrrs t1, mhpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter22h' is RV32 only
+csrrs t1, mhpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter23h' is RV32 only
+csrrs t1, mhpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter24h' is RV32 only
+csrrs t1, mhpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter25h' is RV32 only
+csrrs t1, mhpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter26h' is RV32 only
+csrrs t1, mhpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter27h' is RV32 only
+csrrs t1, mhpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter28h' is RV32 only
+csrrs t1, mhpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter29h' is RV32 only
+csrrs t1, mhpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter30h' is RV32 only
+csrrs t1, mhpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter31h' is RV32 only
diff --git a/llvm/test/MC/RISCV/rv32-only-csr-names.s b/llvm/test/MC/RISCV/rv32-only-csr-names.s
index 5a91aea4c791b20..db88eacf9396bb3 100644
--- a/llvm/test/MC/RISCV/rv32-only-csr-names.s
+++ b/llvm/test/MC/RISCV/rv32-only-csr-names.s
@@ -3,146 +3,146 @@
 
 # The following CSR register names are all RV32 only.
 
-csrrs t1, cycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, timeh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, instreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, hpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, henvcfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, htimedeltah, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mstatush, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, menvcfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mseccfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, pmpcfg1, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg3, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg5, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg7, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg9, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg11, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg13, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg15, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mcycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, minstreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mhpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mhpmevent3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mstateen0h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mstateen1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, hstateen0h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hstateen1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, stimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, vstimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, midelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, miph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, sieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, siph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hidelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hviprio1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hviprio2h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, vsieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, vsiph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, cycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'cycleh' is RV32 only
+csrrs t1, timeh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'timeh' is RV32 only
+csrrs t1, instreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'instreth' is RV32 only
+
+csrrs t1, hpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter3h' is RV32 only
+csrrs t1, hpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter4h' is RV32 only
+csrrs t1, hpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter5h' is RV32 only
+csrrs t1, hpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter6h' is RV32 only
+csrrs t1, hpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter7h' is RV32 only
+csrrs t1, hpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter8h' is RV32 only
+csrrs t1, hpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter9h' is RV32 only
+csrrs t1, hpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter10h' is RV32 only
+csrrs t1, hpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter11h' is RV32 only
+csrrs t1, hpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter12h' is RV32 only
+csrrs t1, hpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter13h' is RV32 only
+csrrs t1, hpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter14h' is RV32 only
+csrrs t1, hpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter15h' is RV32 only
+csrrs t1, hpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter16h' is RV32 only
+csrrs t1, hpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter17h' is RV32 only
+csrrs t1, hpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter18h' is RV32 only
+csrrs t1, hpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter19h' is RV32 only
+csrrs t1, hpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter20h' is RV32 only
+csrrs t1, hpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter21h' is RV32 only
+csrrs t1, hpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter22h' is RV32 only
+csrrs t1, hpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter23h' is RV32 only
+csrrs t1, hpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter24h' is RV32 only
+csrrs t1, hpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter25h' is RV32 only
+csrrs t1, hpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter26h' is RV32 only
+csrrs t1, hpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter27h' is RV32 only
+csrrs t1, hpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter28h' is RV32 only
+csrrs t1, hpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter29h' is RV32 only
+csrrs t1, hpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter30h' is RV32 only
+csrrs t1, hpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter31h' is RV32 only
+
+csrrs t1, henvcfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'henvcfgh' is RV32 only
+
+csrrs t1, htimedeltah, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'htimedeltah' is RV32 only
+
+csrrs t1, mstatush, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstatush' is RV32 only
+
+csrrs t1, menvcfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'menvcfgh' is RV32 only
+
+csrrs t1, mseccfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mseccfgh' is RV32 only
+
+csrrs t1, pmpcfg1, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg1' is RV32 only
+csrrs t1, pmpcfg3, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg3' is RV32 only
+csrrs t1, pmpcfg5, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg5' is RV32 only
+csrrs t1, pmpcfg7, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg7' is RV32 only
+csrrs t1, pmpcfg9, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg9' is RV32 only
+csrrs t1, pmpcfg11, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg11' is RV32 only
+csrrs t1, pmpcfg13, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg13' is RV32 only
+csrrs t1, pmpcfg15, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg15' is RV32 only
+
+csrrs t1, mcycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mcycleh' is RV32 only
+csrrs t1, minstreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'minstreth' is RV32 only
+
+csrrs t1, mhpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter3h' is RV32 only
+csrrs t1, mhpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter4h' is RV32 only
+csrrs t1, mhpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter5h' is RV32 only
+csrrs t1, mhpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter6h' is RV32 only
+csrrs t1, mhpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter7h' is RV32 only
+csrrs t1, mhpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter8h' is RV32 only
+csrrs t1, mhpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter9h' is RV32 only
+csrrs t1, mhpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter10h' is RV32 only
+csrrs t1, mhpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter11h' is RV32 only
+csrrs t1, mhpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter12h' is RV32 only
+csrrs t1, mhpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter13h' is RV32 only
+csrrs t1, mhpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter14h' is RV32 only
+csrrs t1, mhpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter15h' is RV32 only
+csrrs t1, mhpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter16h' is RV32 only
+csrrs t1, mhpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter17h' is RV32 only
+csrrs t1, mhpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter18h' is RV32 only
+csrrs t1, mhpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter19h' is RV32 only
+csrrs t1, mhpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter20h' is RV32 only
+csrrs t1, mhpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter21h' is RV32 only
+csrrs t1, mhpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter22h' is RV32 only
+csrrs t1, mhpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter23h' is RV32 only
+csrrs t1, mhpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter24h' is RV32 only
+csrrs t1, mhpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter25h' is RV32 only
+csrrs t1, mhpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter26h' is RV32 only
+csrrs t1, mhpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter27h' is RV32 only
+csrrs t1, mhpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter28h' is RV32 only
+csrrs t1, mhpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter29h' is RV32 only
+csrrs t1, mhpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter30h' is RV32 only
+csrrs t1, mhpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter31h' is RV32 only
+
+csrrs t1, mhpmevent3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent3h' is RV32 only
+csrrs t1, mhpmevent4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent4h' is RV32 only
+csrrs t1, mhpmevent5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent5h' is RV32 only
+csrrs t1, mhpmevent6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent6h' is RV32 only
+csrrs t1, mhpmevent7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent7h' is RV32 only
+csrrs t1, mhpmevent8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent8h' is RV32 only
+csrrs t1, mhpmevent9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent9h' is RV32 only
+csrrs t1, mhpmevent10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent10h' is RV32 only
+csrrs t1, mhpmevent11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent11h' is RV32 only
+csrrs t1, mhpmevent12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent12h' is RV32 only
+csrrs t1, mhpmevent13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent13h' is RV32 only
+csrrs t1, mhpmevent14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent14h' is RV32 only
+csrrs t1, mhpmevent15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent15h' is RV32 only
+csrrs t1, mhpmevent16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent16h' is RV32 only
+csrrs t1, mhpmevent17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent17h' is RV32 only
+csrrs t1, mhpmevent18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent18h' is RV32 only
+csrrs t1, mhpmevent19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent19h' is RV32 only
+csrrs t1, mhpmevent20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent20h' is RV32 only
+csrrs t1, mhpmevent21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent21h' is RV32 only
+csrrs t1, mhpmevent22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent22h' is RV32 only
+csrrs t1, mhpmevent23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent23h' is RV32 only
+csrrs t1, mhpmevent24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent24h' is RV32 only
+csrrs t1, mhpmevent25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent25h' is RV32 only
+csrrs t1, mhpmevent26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent26h' is RV32 only
+csrrs t1, mhpmevent27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent27h' is RV32 only
+csrrs t1, mhpmevent28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent28h' is RV32 only
+csrrs t1, mhpmevent29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent29h' is RV32 only
+csrrs t1, mhpmevent30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent30h' is RV32 only
+csrrs t1, mhpmevent31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent31h' is RV32 only
+
+csrrs t1, mstateen0h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstateen0h' is RV32 only
+csrrs t1, mstateen1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstateen1h' is RV32 only
+csrrs t1, mstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstateen3h' is RV32 only
+csrrs t1, mstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstateen3h' is RV32 only
+
+csrrs t1, hstateen0h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hstateen0h' is RV32 only
+csrrs t1, hstateen1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hstateen1h' is RV32 only
+csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hstateen3h' is RV32 only
+csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hstateen3h' is RV32 only
+
+csrrs t1, stimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'stimecmph' is RV32 only
+csrrs t1, vstimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'vstimecmph' is RV32 only
+
+csrrs t1, midelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'midelegh' is RV32 only
+csrrs t1, mieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mieh' is RV32 only
+csrrs t1, mvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mvienh' is RV32 only
+csrrs t1, mviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mviph' is RV32 only
+csrrs t1, miph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'miph' is RV32 only
+csrrs t1, sieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'sieh' is RV32 only
+csrrs t1, siph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'siph' is RV32 only
+csrrs t1, hidelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hidelegh' is RV32 only
+csrrs t1, hvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hvienh' is RV32 only
+csrrs t1, hviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hviph' is RV32 only
+csrrs t1, hviprio1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hviprio1h' is RV32 only
+csrrs t1, hviprio2h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hviprio2h' is RV32 only
+csrrs t1, vsieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'vsieh' is RV32 only
+csrrs t1, vsiph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'vsiph' is RV32 only
diff --git a/llvm/test/MC/RISCV/rvv/zvbc.s b/llvm/test/MC/RISCV/rvv/zvbc.s
index 0eb02d153b79b61..324d6e04a308fe5 100644
--- a/llvm/test/MC/RISCV/rvv/zvbc.s
+++ b/llvm/test/MC/RISCV/rvv/zvbc.s
@@ -7,27 +7,36 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-INST
 # RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve64x --mattr=+zvbc %s \
 # RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+# RUN: llvm-mc -triple=riscv32 -show-encoding --mattr=+zve32x --mattr=+experimental-zvbc32e %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv32 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve32x --mattr=+experimental-zvbc32e %s \
+# RUN:        | llvm-objdump -d --mattr=+zve32x --mattr=+experimental-zvbc32e  - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve32x --mattr=+experimental-zvbc32e %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
 
 vclmul.vv v10, v9, v8
 # CHECK-INST: vclmul.vv v10, v9, v8
 # CHECK-ENCODING: [0x57,0x25,0x94,0x32]
-# CHECK-ERROR: instruction requires the following: 'Zvbc' (Vector Carryless Multiplication){{$}}
+# CHECK-ERROR: instruction requires the following: 'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication){{$}}
 # CHECK-UNKNOWN: 32942557 <unknown>
 
 vclmul.vx v10, v9, a0
 # CHECK-INST: vclmul.vx v10, v9, a0
 # CHECK-ENCODING: [0x57,0x65,0x95,0x32]
-# CHECK-ERROR: instruction requires the following: 'Zvbc' (Vector Carryless Multiplication){{$}}
+# CHECK-ERROR: instruction requires the following: 'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication){{$}}
 # CHECK-UNKNOWN: 32956557 <unknown>
 
 vclmulh.vv v10, v9, v8
 # CHECK-INST: vclmulh.vv v10, v9, v8
 # CHECK-ENCODING: [0x57,0x25,0x94,0x36]
-# CHECK-ERROR: instruction requires the following: 'Zvbc' (Vector Carryless Multiplication){{$}}
+# CHECK-ERROR: instruction requires the following: 'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication){{$}}
 # CHECK-UNKNOWN: 36942557 <unknown>
 
 vclmulh.vx v10, v9, a0
 # CHECK-INST: vclmulh.vx v10, v9, a0
 # CHECK-ENCODING: [0x57,0x65,0x95,0x36]
-# CHECK-ERROR: instruction requires the following: 'Zvbc' (Vector Carryless Multiplication){{$}}
+# CHECK-ERROR: instruction requires the following: 'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication){{$}}
 # CHECK-UNKNOWN: 36956557 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/zvkgs.s b/llvm/test/MC/RISCV/rvv/zvkgs.s
new file mode 100644
index 000000000000000..5af4e8bb49ccbb1
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/zvkgs.s
@@ -0,0 +1,21 @@
+# RUN: llvm-mc -triple=riscv32 -show-encoding --mattr=+zve32x --mattr=+experimental-zvkgs %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv32 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve32x --mattr=+experimental-zvkgs %s \
+# RUN:        | llvm-objdump -d --mattr=+zve32x --mattr=+experimental-zvkgs  - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve32x --mattr=+experimental-zvkgs %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vghsh.vs v10, v9, v8
+# CHECK-INST: vghsh.vs v10, v9, v8
+# CHECK-ENCODING: [0x77,0x25,0x94,0x8e]
+# CHECK-ERROR: instruction requires the following: 'Zvkgs' (Vector-Scalar GCM instructions for Cryptography){{$}}
+# CHECK-UNKNOWN: 8e942577 <unknown>
+
+vgmul.vs v10, v9
+# CHECK-INST: vgmul.vs v10, v9
+# CHECK-ENCODING: [0x77,0xa5,0x98,0xa6]
+# CHECK-ERROR: instruction requires the following: 'Zvkgs' (Vector-Scalar GCM instructions for Cryptography){{$}}
+# CHECK-UNKNOWN: a698a577 <unknown>
diff --git a/llvm/test/ObjectYAML/wasm/invalid_section_order.yaml b/llvm/test/ObjectYAML/wasm/invalid_section_order.yaml
index 71767576979b0a3..08319e500e11158 100644
--- a/llvm/test/ObjectYAML/wasm/invalid_section_order.yaml
+++ b/llvm/test/ObjectYAML/wasm/invalid_section_order.yaml
@@ -14,7 +14,7 @@ Sections:
       - Index:           0
         Locals:          []
         Body:            0B
-  # CHECK: yaml2obj: error: out of order section type: 3
+  # CHECK: yaml2obj: error: out of order section type: FUNCTION
   - Type:            FUNCTION
     FunctionTypes:   [ 0 ]
 ...
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
index 0dcd0c2dd50a8e5..9a1a15c8d30df3e 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
@@ -86,12 +86,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
 // CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2([[#LOWER:]]), GIMT_Encode2([[#UPPER:]]), /*)*//*default:*//*Label 4*/ GIMT_Encode4([[#DEFAULT:]]),
-// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(418), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(436), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4(448), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_FABS*//*Label 3*/ GIMT_Encode4(460),
+// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4([[L418:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4([[L436:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4([[L448:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FABS*//*Label 3*/ GIMT_Encode4([[L460:[0-9]+]]),
 // CHECK-NEXT:     // Label 0: @[[#%u, mul(UPPER-LOWER, 4) + 10]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(435), // Rule ID 2 //
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4([[L435:[0-9]+]]), // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       // MIs[0] x
 // CHECK-NEXT:       // No operand predicates
@@ -101,10 +101,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
 // CHECK-NEXT:       // Combiner Rule #2: TwoMatchNoApply
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 5: @435
+// CHECK-NEXT:     // Label 5: @[[L435]]
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @436
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(447), // Rule ID 3 //
+// CHECK-NEXT:     // Label 1: @[[L436]]
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4([[L447:[0-9]+]]), // Rule ID 3 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -112,10 +112,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #3: NoMatchTwoApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
-// CHECK-NEXT:     // Label 6: @447
+// CHECK-NEXT:     // Label 6: @[[L447]]
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @448
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(459), // Rule ID 1 //
+// CHECK-NEXT:     // Label 2: @[[L448]]
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4([[L459:[0-9]+]]), // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -123,10 +123,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #1: TwoMatchTwoApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
-// CHECK-NEXT:     // Label 7: @459
+// CHECK-NEXT:     // Label 7: @[[L459]]
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @460
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(471), // Rule ID 0 //
+// CHECK-NEXT:     // Label 3: @[[L460]]
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4([[L471:[0-9]+]]), // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -134,7 +134,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #0: OneMatchOneApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT:     // Label 8: @471
+// CHECK-NEXT:     // Label 8: @[[L471]]
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     // Label 4: @[[#%u, DEFAULT]]
 // CHECK-NEXT:     GIM_Reject,
diff --git a/llvm/test/TableGen/dump.td b/llvm/test/TableGen/dump.td
index 633e73802ffa3ea..7f021755e2e821a 100644
--- a/llvm/test/TableGen/dump.td
+++ b/llvm/test/TableGen/dump.td
@@ -101,3 +101,6 @@ def Three : BaseClassForSet;
 }
 // CHECK: [[FILE]]:[[@LINE+1]]:1: note: TheSet = [Subset_One, Subset_Two, Three]
 dump "TheSet = " # !repr(TheSet);
+
+// CHECK: [[FILE]]:[[@LINE+1]]:1: note: 0
+dump !repr(!exists<BaseClassForSet>("non-existent-record"));
\ No newline at end of file
diff --git a/llvm/test/TableGen/invalid-macro-name-command-line.td b/llvm/test/TableGen/invalid-macro-name-command-line.td
new file mode 100644
index 000000000000000..0d2307997ebe545
--- /dev/null
+++ b/llvm/test/TableGen/invalid-macro-name-command-line.td
@@ -0,0 +1,9 @@
+// RUN: not llvm-tblgen %s -DMACRO=1 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-1
+// RUN: not llvm-tblgen %s -D0MAC 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-2
+// RUN: not llvm-tblgen %s -D_MAC# 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-3
+// RUN: not llvm-tblgen %s -D 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-4
+
+// CHECK-TEST-1: error: Invalid macro name `MACRO=1` specified on command line
+// CHECK-TEST-2: error: Invalid macro name `0MAC` specified on command line
+// CHECK-TEST-3: error: Invalid macro name `_MAC#` specified on command line
+// CHECK-TEST-4: for the -D option: requires a value!
diff --git a/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll b/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll
new file mode 100644
index 000000000000000..a7622143c2e7366
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll
@@ -0,0 +1,588 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
+; RUN: opt -passes=attributor -S < %s | FileCheck %s --check-prefixes=CHECK
+
+define amdgpu_kernel void @__omp_offloading_fd00_2c00523__ZN11qmcplusplus7ompBLAS9gemv_implIfEEiRiciiT_PKS3_iS5_iS3_PS3_i_l383() {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
+; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_fd00_2c00523__ZN11qmcplusplus7ompBLAS9gemv_implIfEEiRiciiT_PKS3_iS5_iS3_PS3_i_l383
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca [0 x [0 x float]], i32 0, align 8, addrspace(5)
+; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr addrspace(5) [[TMP1]] to ptr
+; CHECK-NEXT:    store ptr [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr nocapture nofree noundef readonly align 8 dereferenceable_or_null(8) [[TMP2]], i1 noundef false)
+; CHECK-NEXT:    ret void
+;
+  %1 = alloca [0 x [0 x float]], i32 0, align 8, addrspace(5)
+  %2 = addrspacecast ptr addrspace(5) %1 to ptr
+  store ptr %2, ptr addrspace(5) %1, align 8
+  %3 = call fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr %2, i1 false)
+  ret void
+}
+
+define fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr %0, i1 %1) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
+; CHECK-LABEL: define {{[^@]+}}@__kmpc_nvptx_parallel_reduce_nowait_v2
+; CHECK-SAME: (ptr nocapture nofree nonnull readonly align 8 dereferenceable(8) [[TMP0:%.*]], i1 noundef [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP3:%.*]], label [[TMP79:%.*]]
+; CHECK:       3:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP4:%.*]], label [[TMP6:%.*]]
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP5]], align 8
+; CHECK-NEXT:    br label [[TMP77:%.*]]
+; CHECK:       6:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP7:%.*]], label [[TMP9:%.*]]
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP8]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       9:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP10:%.*]], label [[TMP12:%.*]]
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP11]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       12:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP13:%.*]], label [[TMP15:%.*]]
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP14]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       15:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP16:%.*]], label [[TMP18:%.*]]
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP17]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       18:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP19:%.*]], label [[TMP21:%.*]]
+; CHECK:       19:
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP20]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       21:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP22:%.*]], label [[TMP24:%.*]]
+; CHECK:       22:
+; CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP23]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       24:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP25:%.*]], label [[TMP27:%.*]]
+; CHECK:       25:
+; CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP26]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       27:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP28:%.*]], label [[TMP30:%.*]]
+; CHECK:       28:
+; CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP29]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       30:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP31:%.*]], label [[TMP33:%.*]]
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP32]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       33:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP34:%.*]], label [[TMP36:%.*]]
+; CHECK:       34:
+; CHECK-NEXT:    [[TMP35:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP35]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       36:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP37:%.*]], label [[TMP39:%.*]]
+; CHECK:       37:
+; CHECK-NEXT:    [[TMP38:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP38]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       39:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP40:%.*]], label [[TMP42:%.*]]
+; CHECK:       40:
+; CHECK-NEXT:    [[TMP41:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP41]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       42:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP43:%.*]], label [[TMP45:%.*]]
+; CHECK:       43:
+; CHECK-NEXT:    [[TMP44:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP44]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       45:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP46:%.*]], label [[TMP48:%.*]]
+; CHECK:       46:
+; CHECK-NEXT:    [[TMP47:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP47]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       48:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP49:%.*]], label [[TMP51:%.*]]
+; CHECK:       49:
+; CHECK-NEXT:    [[TMP50:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP50]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       51:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP52:%.*]], label [[TMP54:%.*]]
+; CHECK:       52:
+; CHECK-NEXT:    [[TMP53:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP53]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       54:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP55:%.*]], label [[TMP57:%.*]]
+; CHECK:       55:
+; CHECK-NEXT:    [[TMP56:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP56]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       57:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP58:%.*]], label [[TMP60:%.*]]
+; CHECK:       58:
+; CHECK-NEXT:    [[TMP59:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP59]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       60:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP61:%.*]], label [[TMP63:%.*]]
+; CHECK:       61:
+; CHECK-NEXT:    [[TMP62:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP62]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       63:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP64:%.*]], label [[TMP66:%.*]]
+; CHECK:       64:
+; CHECK-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP65]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       66:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP67:%.*]], label [[TMP69:%.*]]
+; CHECK:       67:
+; CHECK-NEXT:    [[TMP68:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP68]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       69:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP70:%.*]], label [[TMP72:%.*]]
+; CHECK:       70:
+; CHECK-NEXT:    [[TMP71:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP71]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       72:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP73:%.*]], label [[TMP75:%.*]]
+; CHECK:       73:
+; CHECK-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP74]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       75:
+; CHECK-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP76]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       77:
+; CHECK-NEXT:    [[TMP78:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP78]], align 4
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP80:%.*]], label [[TMP82:%.*]]
+; CHECK:       79:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       80:
+; CHECK-NEXT:    [[TMP81:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP81]], align 8
+; CHECK-NEXT:    ret i32 0
+; CHECK:       82:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP83:%.*]], label [[TMP85:%.*]]
+; CHECK:       83:
+; CHECK-NEXT:    [[TMP84:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP84]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       85:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP86:%.*]], label [[TMP88:%.*]]
+; CHECK:       86:
+; CHECK-NEXT:    [[TMP87:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP87]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       88:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP89:%.*]], label [[TMP91:%.*]]
+; CHECK:       89:
+; CHECK-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP90]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       91:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP92:%.*]], label [[TMP94:%.*]]
+; CHECK:       92:
+; CHECK-NEXT:    [[TMP93:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP93]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       94:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP95:%.*]], label [[TMP97:%.*]]
+; CHECK:       95:
+; CHECK-NEXT:    [[TMP96:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP96]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       97:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP98:%.*]], label [[TMP100:%.*]]
+; CHECK:       98:
+; CHECK-NEXT:    [[TMP99:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP99]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       100:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP101:%.*]], label [[TMP103:%.*]]
+; CHECK:       101:
+; CHECK-NEXT:    [[TMP102:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP102]], align 8
+; CHECK-NEXT:    ret i32 0
+; CHECK:       103:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP104:%.*]], label [[TMP106:%.*]]
+; CHECK:       104:
+; CHECK-NEXT:    [[TMP105:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP105]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       106:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP107:%.*]], label [[TMP109:%.*]]
+; CHECK:       107:
+; CHECK-NEXT:    [[TMP108:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP108]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       109:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP110:%.*]], label [[TMP112:%.*]]
+; CHECK:       110:
+; CHECK-NEXT:    [[TMP111:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP111]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       112:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP113:%.*]], label [[TMP115:%.*]]
+; CHECK:       113:
+; CHECK-NEXT:    [[TMP114:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP114]], align 8
+; CHECK-NEXT:    ret i32 0
+; CHECK:       115:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP116:%.*]], label [[TMP118:%.*]]
+; CHECK:       116:
+; CHECK-NEXT:    [[TMP117:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP117]], align 8
+; CHECK-NEXT:    ret i32 0
+; CHECK:       118:
+; CHECK-NEXT:    [[TMP119:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP119]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+  br i1 %1, label %3, label %88
+
+3:                                                ; preds = %2
+  %4 = load ptr, ptr %0, align 8
+  br i1 %1, label %5, label %7
+
+5:                                                ; preds = %3
+  %6 = load ptr, ptr %0, align 8
+  store i64 0, ptr %6, align 8
+  br label %79
+
+7:                                                ; preds = %3
+  br i1 %1, label %8, label %11
+
+8:                                                ; preds = %7
+  %9 = load i32, ptr %4, align 4
+  %10 = load ptr, ptr %0, align 8
+  store i32 0, ptr %10, align 4
+  br label %79
+
+11:                                               ; preds = %7
+  br i1 %1, label %12, label %14
+
+12:                                               ; preds = %11
+  %13 = load ptr, ptr %0, align 8
+  store i32 0, ptr %13, align 4
+  br label %79
+
+14:                                               ; preds = %11
+  br i1 %1, label %15, label %17
+
+15:                                               ; preds = %14
+  %16 = load ptr, ptr %0, align 8
+  store i32 0, ptr %16, align 4
+  br label %79
+
+17:                                               ; preds = %14
+  br i1 %1, label %18, label %20
+
+18:                                               ; preds = %17
+  %19 = load ptr, ptr %0, align 8
+  store i32 0, ptr %19, align 4
+  br label %79
+
+20:                                               ; preds = %17
+  br i1 %1, label %21, label %23
+
+21:                                               ; preds = %20
+  %22 = load ptr, ptr %0, align 8
+  store i32 0, ptr %22, align 4
+  br label %79
+
+23:                                               ; preds = %20
+  br i1 %1, label %24, label %26
+
+24:                                               ; preds = %23
+  %25 = load ptr, ptr %0, align 8
+  store i32 0, ptr %25, align 4
+  br label %79
+
+26:                                               ; preds = %23
+  br i1 %1, label %27, label %29
+
+27:                                               ; preds = %26
+  %28 = load ptr, ptr %0, align 8
+  store i32 0, ptr %28, align 8
+  br label %79
+
+29:                                               ; preds = %26
+  br i1 %1, label %30, label %32
+
+30:                                               ; preds = %29
+  %31 = load ptr, ptr %0, align 8
+  store i32 0, ptr %31, align 4
+  br label %79
+
+32:                                               ; preds = %29
+  br i1 %1, label %33, label %35
+
+33:                                               ; preds = %32
+  %34 = load ptr, ptr %0, align 8
+  store i32 0, ptr %34, align 4
+  br label %79
+
+35:                                               ; preds = %32
+  br i1 %1, label %36, label %38
+
+36:                                               ; preds = %35
+  %37 = load ptr, ptr %0, align 8
+  store i32 0, ptr %37, align 4
+  br label %79
+
+38:                                               ; preds = %35
+  br i1 %1, label %39, label %41
+
+39:                                               ; preds = %38
+  %40 = load ptr, ptr %0, align 8
+  store i64 0, ptr %40, align 8
+  br label %79
+
+41:                                               ; preds = %38
+  br i1 %1, label %42, label %44
+
+42:                                               ; preds = %41
+  %43 = load ptr, ptr %0, align 8
+  store i64 0, ptr %43, align 8
+  br label %79
+
+44:                                               ; preds = %41
+  br i1 %1, label %45, label %47
+
+45:                                               ; preds = %44
+  %46 = load ptr, ptr %0, align 8
+  store i32 0, ptr %46, align 4
+  br label %79
+
+47:                                               ; preds = %44
+  br i1 %1, label %48, label %50
+
+48:                                               ; preds = %47
+  %49 = load ptr, ptr %0, align 8
+  store i32 0, ptr %49, align 4
+  br label %79
+
+50:                                               ; preds = %47
+  br i1 %1, label %51, label %53
+
+51:                                               ; preds = %50
+  %52 = load ptr, ptr %0, align 8
+  store i64 0, ptr %52, align 8
+  br label %79
+
+53:                                               ; preds = %50
+  br i1 %1, label %54, label %56
+
+54:                                               ; preds = %53
+  %55 = load ptr, ptr %0, align 8
+  store i64 0, ptr %55, align 8
+  br label %79
+
+56:                                               ; preds = %53
+  br i1 %1, label %57, label %59
+
+57:                                               ; preds = %56
+  %58 = load ptr, ptr %0, align 8
+  store i32 0, ptr %58, align 4
+  br label %79
+
+59:                                               ; preds = %56
+  br i1 %1, label %60, label %62
+
+60:                                               ; preds = %59
+  %61 = load ptr, ptr %0, align 8
+  store i32 0, ptr %61, align 4
+  br label %79
+
+62:                                               ; preds = %59
+  br i1 %1, label %63, label %65
+
+63:                                               ; preds = %62
+  %64 = load ptr, ptr %0, align 8
+  store i64 0, ptr %64, align 8
+  br label %79
+
+65:                                               ; preds = %62
+  br i1 %1, label %66, label %68
+
+66:                                               ; preds = %65
+  %67 = load ptr, ptr %0, align 8
+  store i64 0, ptr %67, align 8
+  br label %79
+
+68:                                               ; preds = %65
+  br i1 %1, label %69, label %71
+
+69:                                               ; preds = %68
+  %70 = load ptr, ptr %0, align 8
+  store i32 0, ptr %70, align 4
+  br label %79
+
+71:                                               ; preds = %68
+  br i1 %1, label %72, label %74
+
+72:                                               ; preds = %71
+  %73 = load ptr, ptr %0, align 8
+  store i32 0, ptr %73, align 4
+  br label %79
+
+74:                                               ; preds = %71
+  br i1 %1, label %75, label %77
+
+75:                                               ; preds = %74
+  %76 = load ptr, ptr %0, align 8
+  store i32 0, ptr %76, align 4
+  br label %79
+
+77:                                               ; preds = %74
+  %78 = load ptr, ptr %0, align 8
+  store i32 0, ptr %78, align 4
+  br label %79
+
+79:                                               ; preds = %77, %75, %72, %69, %66, %63, %60, %57, %54, %51, %48, %45, %42, %39, %36, %33, %30, %27, %24, %21, %18, %15, %12, %8, %5
+  %80 = load ptr, ptr %0, align 8
+  %81 = load i32, ptr %80, align 4
+  %82 = load ptr, ptr %0, align 8
+  store i32 0, ptr %82, align 4
+  %83 = load ptr, ptr %0, align 8
+  %84 = getelementptr i8, ptr %83, i64 4
+  %85 = load ptr, ptr %0, align 8
+  %86 = getelementptr i8, ptr %85, i64 4
+  %87 = load ptr, ptr %0, align 8
+  br i1 %1, label %91, label %93
+
+88:                                               ; preds = %2
+  %89 = load ptr, ptr %0, align 8
+  %90 = load i64, ptr %89, align 8
+  ret i32 0
+
+91:                                               ; preds = %79
+  %92 = load ptr, ptr %0, align 8
+  store i64 0, ptr %92, align 8
+  ret i32 0
+
+93:                                               ; preds = %79
+  br i1 %1, label %94, label %96
+
+94:                                               ; preds = %93
+  %95 = load ptr, ptr %0, align 8
+  store i32 0, ptr %95, align 4
+  ret i32 0
+
+96:                                               ; preds = %93
+  br i1 %1, label %97, label %99
+
+97:                                               ; preds = %96
+  %98 = load ptr, ptr %0, align 8
+  store i32 0, ptr %98, align 4
+  ret i32 0
+
+99:                                               ; preds = %96
+  br i1 %1, label %100, label %102
+
+100:                                              ; preds = %99
+  %101 = load ptr, ptr %0, align 8
+  store i32 0, ptr %101, align 4
+  ret i32 0
+
+102:                                              ; preds = %99
+  br i1 %1, label %103, label %105
+
+103:                                              ; preds = %102
+  %104 = load ptr, ptr %0, align 8
+  store i32 0, ptr %104, align 4
+  ret i32 0
+
+105:                                              ; preds = %102
+  br i1 %1, label %106, label %108
+
+106:                                              ; preds = %105
+  %107 = load ptr, ptr %0, align 8
+  store i32 0, ptr %107, align 4
+  ret i32 0
+
+108:                                              ; preds = %105
+  br i1 %1, label %109, label %111
+
+109:                                              ; preds = %108
+  %110 = load ptr, ptr %0, align 8
+  store i32 0, ptr %110, align 4
+  ret i32 0
+
+111:                                              ; preds = %108
+  br i1 %1, label %112, label %114
+
+112:                                              ; preds = %111
+  %113 = load ptr, ptr %0, align 8
+  store i32 0, ptr %113, align 8
+  ret i32 0
+
+114:                                              ; preds = %111
+  br i1 %1, label %115, label %117
+
+115:                                              ; preds = %114
+  %116 = load ptr, ptr %0, align 8
+  store i32 0, ptr %116, align 4
+  ret i32 0
+
+117:                                              ; preds = %114
+  br i1 %1, label %118, label %121
+
+118:                                              ; preds = %117
+  %119 = load i32, ptr %87, align 4
+  %120 = load ptr, ptr %0, align 8
+  store i32 0, ptr %120, align 4
+  ret i32 0
+
+121:                                              ; preds = %117
+  br i1 %1, label %122, label %124
+
+122:                                              ; preds = %121
+  %123 = load ptr, ptr %0, align 8
+  store i32 0, ptr %123, align 4
+  ret i32 0
+
+124:                                              ; preds = %121
+  br i1 %1, label %125, label %127
+
+125:                                              ; preds = %124
+  %126 = load ptr, ptr %0, align 8
+  store i64 0, ptr %126, align 8
+  ret i32 0
+
+127:                                              ; preds = %124
+  br i1 %1, label %128, label %130
+
+128:                                              ; preds = %127
+  %129 = load ptr, ptr %0, align 8
+  store i64 0, ptr %129, align 8
+  ret i32 0
+
+130:                                              ; preds = %127
+  %131 = load ptr, ptr %0, align 8
+  store i32 0, ptr %131, align 4
+  ret i32 0
+}
+
+; uselistorder directives
+uselistorder i32 0, { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 0 }
diff --git a/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll b/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
index fda72a6e31a0c78..518ed97f42bc107 100644
--- a/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
+++ b/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
@@ -116,6 +116,7 @@ cond.end:                                         ; preds = %cond.true, %entry
 ; CHECK-LABEL: define {{[^@]+}}@_ZN6Kokkos4Impl14SubviewExtentsILj2ELj1EE3setIJLm0ELm0EEJiEEEbjjRKNS0_13ViewDimensionIJXspT_EEEENS0_5ALL_tEDpT0_.internalized
 ; CHECK-SAME: (ptr nocapture writeonly [[THIS:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[THIS]] to ptr addrspace(5)
 ; CHECK-NEXT:    ret i1 false
 ;
 ;
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll b/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
new file mode 100644
index 000000000000000..a019679e65905de
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
@@ -0,0 +1,858 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=x86_64-unknown-unknown --loop-simplify -codegenprepare -S | FileCheck %s
+
+declare void @use.i32(i32)
+declare void @use.2xi64(<2 x i64>)
+declare void @do_stuff0()
+declare void @do_stuff1()
+declare i1 @get.i1()
+declare i32 @get.i32()
+
+define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_nested2(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY_TAIL:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[COND0:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND0]], label %[[FOR_BODY0:.*]], label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY0]]:
+; CHECK-NEXT:    [[COND1:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND1]], label %[[FOR_BODY1:.*]], label %[[FOR_BODY2:.*]]
+; CHECK:       [[FOR_BODY2]]:
+; CHECK-NEXT:    [[COND2:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND2]], label %[[FOR_BODY1]], label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY1]]:
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    br label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY_TAIL]]:
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body.tail ], [ 0, %entry ]
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body.tail
+for.body0:
+  %cond1 = call i1 @get.i1()
+  br i1 %cond1, label %for.body1, label %for.body2
+for.body2:
+  %cond2 = call i1 @get.i1()
+  br i1 %cond2, label %for.body1, label %for.body.tail
+for.body1:
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  br label %for.body.tail
+for.body.tail:
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr3(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_bad_incr3(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[COND0:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND0]], label %[[FOR_BODY0:.*]], label %[[FOR_BODY2:.*]]
+; CHECK:       [[FOR_BODY0]]:
+; CHECK-NEXT:    [[COND1:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[VAL]], 1
+; CHECK-NEXT:    br i1 [[COND1]], label %[[FOR_BODY1:.*]], label %[[FOR_BODY_TAIL:.*]]
+; CHECK:       [[FOR_BODY2]]:
+; CHECK-NEXT:    [[COND2:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND2]], label %[[FOR_BODY1]], label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY1]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY0]] ], [ 0, %[[FOR_BODY2]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    br label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY_TAIL]]:
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body2
+for.body0:
+  %cond1 = call i1 @get.i1()
+  %val = call i32 @get.i32()
+  %inc = add nuw i32 %val, 1
+  br i1 %cond1, label %for.body1, label %for.body.tail
+for.body2:
+  %cond2 = call i1 @get.i1()
+  br i1 %cond2, label %for.body1, label %for.body.tail
+for.body1:
+  %i.04 = phi i32 [ %inc, %for.body0], [ 0, %for.body2 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  br label %for.body.tail
+for.body.tail:
+  %exitcond.not = call i1 @get.i1()
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_vec(
+; CHECK-SAME: <2 x i64> [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi <2 x i64> [ [[INC:%.*]], %[[FOR_BODY]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem <2 x i64> [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.2xi64(<2 x i64> [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw <2 x i64> [[I_04]], <i64 1, i64 1>
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi <2 x i64> [ %inc, %for.body ], [ zeroinitializer, %entry ]
+  %rem = urem <2 x i64> %i.04, %rem_amt
+  tail call void @use.2xi64(<2 x i64> %rem)
+  %inc = add nuw <2 x i64> %i.04, <i64 1, i64 1>
+  %exitcond.not = call i1 @get.i1()
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_bad_incr(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_03:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY_TAIL:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[COND0:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND0]], label %[[FOR_BODY0:.*]], label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY0]]:
+; CHECK-NEXT:    [[SOME_VAL:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    br label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY_TAIL]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[I_03]], %[[FOR_BODY]] ], [ [[SOME_VAL]], %[[FOR_BODY0]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.03 = phi i32 [ %inc, %for.body.tail ], [ 0, %entry ]
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body.tail
+for.body0:
+  %some_val = call i32 @get.i32()
+  br label %for.body.tail
+
+for.body.tail:
+  %i.04 = phi i32 [ %i.03, %for.body ], [ %some_val, %for.body0 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_second_acc(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC2:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[INC2]] = add nuw i32 [[I_05]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp ugt i32 [[INC2]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ult i32 %N, 2
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %i.05 = phi i32 [ %inc2, %for.body ], [ 1, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %inc2 = add nuw i32 %i.05, 2
+  %exitcond.not = icmp ugt i32 %inc2, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_srem(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_srem(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = srem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_missing_nuw(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_missing_nuw(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nsw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr2(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_bad_incr2(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 2
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_non_zero_entry4(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_non_zero_entry4(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 4, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 4, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_skip_const_rem_amt(i32 %N) nounwind {
+; CHECK-LABEL: define void @simple_urem_skip_const_rem_amt(
+; CHECK-SAME: i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 4, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], 19
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 4, %entry ]
+  %rem = urem i32 %i.04, 19
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_no_preheader_non_canonical(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_no_preheader_non_canonical(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY1:.*]]
+; CHECK:       [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY1]]:
+; CHECK-NEXT:    br label %[[FOR_BODY_PREHEADER]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    [[I_04_PH:%.*]] = phi i32 [ 1, %[[FOR_BODY1]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[I_04_PH]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.body0, label %for.body1
+
+for.cond.cleanup:
+  ret void
+
+for.body0:
+  br label %for.body
+
+for.body1:
+  br label %for.body
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %for.body0 ], [ 1, %for.body1 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_multi_latch_non_canonical(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[INC]], %[[FOR_BODY0:.*]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[COND:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    call void @do_stuff0()
+; CHECK-NEXT:    br i1 [[COND]], label %[[FOR_BODY0]], label %[[FOR_BODY]]
+; CHECK:       [[FOR_BODY0]]:
+; CHECK-NEXT:    call void @do_stuff1()
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %inc, %for.body0 ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %cond = call i1 @get.i1()
+  call void @do_stuff0()
+  br i1 %cond, label %for.body0, label %for.body
+for.body0:
+  call void @do_stuff1()
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_loop(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_bad_loop(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[FOR_COND:.*]], label %[[HALFWAY:.*]]
+; CHECK:       [[FOR_COND]]:
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ [[INC:%.*]], %[[HALFWAY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[I_0]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[I_0]], 1
+; CHECK-NEXT:    call void @use.i32(i32 [[XOR]])
+; CHECK-NEXT:    br label %[[HALFWAY]]
+; CHECK:       [[HALFWAY]]:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[I_0]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_1]], [[REM_AMT]]
+; CHECK-NEXT:    call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_1]], 1
+; CHECK-NEXT:    br label %[[FOR_COND]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %call = call i32 @get.i32()
+  %tobool.not = icmp eq i32 %call, 0
+  br i1 %tobool.not, label %for.cond, label %halfway
+
+for.cond:
+  %i.0 = phi i32 [ %inc, %halfway ], [ 0, %entry ]
+  %cmp = icmp ult i32 %i.0, %N
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %xor = xor i32 %i.0, 1
+  call void @use.i32(i32 %xor)
+  br label %halfway
+
+halfway:
+  %i.1 = phi i32 [ poison, %entry ], [ %i.0, %for.body ]
+  %rem = urem i32 %i.1, %rem_amt
+  call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.1, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+define void @simple_urem_fail_intermediate_inc(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_intermediate_inc(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[INC2:%.*]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[INC2]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %inc2 = add nuw i32 %i.04, 1
+  %rem = urem i32 %inc2, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @weird_loop(i64 %sub.ptr.div.i56) personality ptr null {
+; CHECK-LABEL: define void @weird_loop(
+; CHECK-SAME: i64 [[SUB_PTR_DIV_I56:%.*]]) personality ptr null {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[ADD74_US:%.*]] = add nuw i64 0, 1
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[REM_US:%.*]] = urem i64 0, [[SUB_PTR_DIV_I56]]
+; CHECK-NEXT:    br label %[[FOR_BODY]]
+;
+entry:
+  br label %for.preheader
+
+for.preheader:
+  %i57.0540.us = phi i64 [ 0, %entry ], [ %add74.us, %for.body ]
+  %add74.us = add nuw i64 %i57.0540.us, 1
+  br label %for.body
+
+for.body:
+  %rem.us = urem i64 %i57.0540.us, %sub.ptr.div.i56
+  br i1 false, label %for.preheader, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add(i32 %N, i32 %rem_amt_in) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_add(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT_IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[REM_AMT:%.*]] = or i32 [[REM_AMT_IN]], 16
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = add nuw i32 [[I_04]], 5
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %rem_amt = or i32 %rem_amt_in, 16
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add nuw i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw(i32 %N, i32 %rem_amt_in) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT_IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[REM_AMT:%.*]] = or i32 [[REM_AMT_IN]], 16
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = add i32 [[I_04]], 5
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %rem_amt = or i32 %rem_amt_in, 16
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = add nuw i32 [[I_04]], 5
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add nuw i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_sub(i32 %N, i32 %rem_amt, i32 %start) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_sub(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ule i32 [[N]], [[START]]
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = sub nuw i32 [[I_04]], [[START]]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ule i32 %N, %start
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %start, %entry ]
+  %i_with_off = sub nuw i32 %i.04, %start
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_sub_no_simplfy(i32 %N, i32 %rem_amt, i32 %start) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_sub_no_simplfy(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ule i32 [[N]], [[START]]
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = sub nuw i32 [[I_04]], 2
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ule i32 %N, %start
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %start, %entry ]
+  %i_with_off = sub nuw i32 %i.04, 2
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-bpf.ll b/llvm/test/Transforms/DCE/intrinsics-bpf.ll
new file mode 100644
index 000000000000000..135588ba21cbb15
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-bpf.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare i64 @llvm.bpf.load.half(ptr, i64)
+declare i64 @llvm.bpf.load.word(ptr, i64)
+declare i64 @llvm.bpf.load.byte(ptr, i64)
+
+define void @test_bpf_load_half(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_bpf_load_half(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.bpf.load.half(ptr %a, i64 %b)
+  ret void
+}
+
+define void @test_bpf_load_word(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_bpf_load_word(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.bpf.load.word(ptr %a, i64 %b)
+  ret void
+}
+
+define void @test_bpf_load_byte(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_bpf_load_byte(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.bpf.load.byte(ptr %a, i64 %b)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-loongarch.ll b/llvm/test/Transforms/DCE/intrinsics-loongarch.ll
new file mode 100644
index 000000000000000..665069091a48de6
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-loongarch.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare <16 x i8> @llvm.loongarch.lsx.vld(ptr, i32)
+declare <16 x i8> @llvm.loongarch.lsx.vldx(ptr, i64)
+declare <16 x i8> @llvm.loongarch.lsx.vldrepl.b(ptr, i32)
+declare <8 x i16> @llvm.loongarch.lsx.vldrepl.h(ptr, i32)
+declare <4 x i32> @llvm.loongarch.lsx.vldrepl.w(ptr, i32)
+declare <2 x i64> @llvm.loongarch.lsx.vldrepl.d(ptr, i32)
+
+declare <32 x i8> @llvm.loongarch.lasx.xvld(ptr, i32)
+declare <32 x i8> @llvm.loongarch.lasx.xvldx(ptr, i64)
+declare <32 x i8> @llvm.loongarch.lasx.xvldrepl.b(ptr, i32)
+declare <16 x i16> @llvm.loongarch.lasx.xvldrepl.h(ptr, i32)
+declare <8 x i32> @llvm.loongarch.lasx.xvldrepl.w(ptr, i32)
+declare <4 x i64> @llvm.loongarch.lasx.xvldrepl.d(ptr, i32)
+
+define void @test_loongarch_lsx_vld(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vld(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.loongarch.lsx.vld(ptr %a, i32 8)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldx(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldx(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.loongarch.lsx.vldx(ptr %a, i64 %b)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldrepl_b(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldrepl_b(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.loongarch.lsx.vldrepl.b(ptr %a, i32 12)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldrepl_h(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldrepl_h(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <8 x i16> @llvm.loongarch.lsx.vldrepl.h(ptr %a, i32 12)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldrepl_w(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldrepl_w(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i32> @llvm.loongarch.lsx.vldrepl.w(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldrepl_d(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldrepl_d(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <2 x i64> @llvm.loongarch.lsx.vldrepl.d(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvld(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvld(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <32 x i8> @llvm.loongarch.lasx.xvld(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldx(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldx(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <32 x i8> @llvm.loongarch.lasx.xvldx(ptr %a, i64 %b)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldrepl_b(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldrepl_b(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <32 x i8> @llvm.loongarch.lasx.xvldrepl.b(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldrepl_h(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldrepl_h(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i16> @llvm.loongarch.lasx.xvldrepl.h(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldrepl_w(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldrepl_w(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <8 x i32> @llvm.loongarch.lasx.xvldrepl.w(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldrepl_d(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldrepl_d(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i64> @llvm.loongarch.lasx.xvldrepl.d(ptr %a, i32 4)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-mips.ll b/llvm/test/Transforms/DCE/intrinsics-mips.ll
new file mode 100644
index 000000000000000..22d709e21e061b5
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-mips.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare i32 @llvm.mips.rddsp(i32)
+declare <4 x i8> @llvm.mips.pick.qb(<4 x i8>, <4 x i8>)
+declare <2 x i16> @llvm.mips.pick.ph(<2 x i16>, <2 x i16>)
+declare i32 @llvm.mips.bposge32()
+declare i32 @llvm.mips.lbux(ptr, i32)
+declare i32 @llvm.mips.lhx(ptr, i32)
+declare i32 @llvm.mips.lwx(ptr, i32)
+declare <16 x i8> @llvm.mips.ld.b(ptr, i32)
+declare <8 x i16> @llvm.mips.ld.h(ptr, i32)
+declare <4 x i32> @llvm.mips.ld.w(ptr, i32)
+declare <2 x i64> @llvm.mips.ld.d(ptr, i32)
+declare <2 x i64> @llvm.mips.ldr.d(ptr, i32)
+declare <4 x i32> @llvm.mips.ldr.w(ptr, i32)
+
+define void @test_mips_rddsp() {
+; CHECK-LABEL: define void @test_mips_rddsp() {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.rddsp(i32 4)
+  ret void
+}
+
+define void @test_llvm_mips_pick_qb(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: define void @test_llvm_mips_pick_qb(
+; CHECK-SAME: <4 x i8> [[A:%.*]], <4 x i8> [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i8> @llvm.mips.pick.qb(<4 x i8> %a, <4 x i8> %b)
+  ret void
+}
+
+define void @test_llvm_mips_pick_ph(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: define void @test_llvm_mips_pick_ph(
+; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <2 x i16> @llvm.mips.pick.ph(<2 x i16> %a, <2 x i16> %b)
+  ret void
+}
+
+define void @test_llvm_mips_bposge32() {
+; CHECK-LABEL: define void @test_llvm_mips_bposge32() {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.bposge32()
+  ret void
+}
+
+define void @test_llvm_mips_lbux(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_lbux(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.lbux(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_lhx(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_lhx(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.lhx(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_lwx(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_lwx(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.lwx(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ld_b(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ld_b(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.mips.ld.b(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ld_h(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ld_h(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <8 x i16> @llvm.mips.ld.h(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ld_w(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ld_w(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i32> @llvm.mips.ld.w(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ld_d(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ld_d(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <2 x i64> @llvm.mips.ld.d(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ldr_d(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ldr_d(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <2 x i64> @llvm.mips.ldr.d(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ldr_w(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ldr_w(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i32> @llvm.mips.ldr.w(ptr %a, i32 %b)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-nvvm.ll b/llvm/test/Transforms/DCE/intrinsics-nvvm.ll
new file mode 100644
index 000000000000000..f0b5c2a5920f723
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-nvvm.ll
@@ -0,0 +1,759 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+declare i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i16 @llvm.nvvm.ldu.global.i.i16.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i32 @llvm.nvvm.ldu.global.i.i32.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i64 @llvm.nvvm.ldu.global.i.i64.p1(ptr addrspace(1) %ptr, i32 %align)
+declare ptr @llvm.nvvm.ldu.global.p.p1(ptr addrspace(1) %ptr, i32 %align)
+declare float @llvm.nvvm.ldu.global.f.f32.p1(ptr addrspace(1) %ptr, i32 %align)
+declare double @llvm.nvvm.ldu.global.f.f64.p1(ptr addrspace(1) %ptr, i32 %align)
+declare half @llvm.nvvm.ldu.global.f.f16.p1(ptr addrspace(1) %ptr, i32 %align)
+declare <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 %align)
+
+declare i8 @llvm.nvvm.ldg.global.i.i8.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i16 @llvm.nvvm.ldg.global.i.i16.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i64 @llvm.nvvm.ldg.global.i.i64.p1(ptr addrspace(1) %ptr, i32 %align)
+declare ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %ptr, i32 %align)
+declare float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %ptr, i32 %align)
+declare double @llvm.nvvm.ldg.global.f.f64.p1(ptr addrspace(1) %ptr, i32 %align)
+declare half @llvm.nvvm.ldg.global.f.f16.p1(ptr addrspace(1) %ptr, i32 %align)
+declare <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 %align)
+
+define void @test_ldu_i8_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_i8_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldu_i16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_i16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i16 @llvm.nvvm.ldu.global.i.i16.p1(ptr addrspace(1) %ptr, i32 2)
+  ret void
+}
+
+define void @test_ldu_i32_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_i32_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i32 @llvm.nvvm.ldu.global.i.i32.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldu_i64_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_i64_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i64 @llvm.nvvm.ldu.global.i.i64.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldu_p_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_p_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call ptr @llvm.nvvm.ldu.global.p.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldu_f32_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_f32_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call float @llvm.nvvm.ldu.global.f.f32.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldu_f64_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_f64_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call double @llvm.nvvm.ldu.global.f.f64.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldu_f16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_f16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call half @llvm.nvvm.ldu.global.f.f16.p1(ptr addrspace(1) %ptr, i32 2)
+  ret void
+}
+
+define void @test_ldu_v2f16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_v2f16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldg_i8_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_i8_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i8 @llvm.nvvm.ldg.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldg_i16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_i16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i16 @llvm.nvvm.ldg.global.i.i16.p1(ptr addrspace(1) %ptr, i32 2)
+  ret void
+}
+
+define void @test_ldg_i32_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_i32_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldg_i64_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_i64_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i64 @llvm.nvvm.ldg.global.i.i64.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldg_p_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_p_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldg_f32_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_f32_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldg_f64_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_f64_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call double @llvm.nvvm.ldg.global.f.f64.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldg_f16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_f16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call half @llvm.nvvm.ldg.global.f.f16.p1(ptr addrspace(1) %ptr, i32 2)
+  ret void
+}
+
+define void @test_ldg_v2f16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_v2f16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-systemz.ll b/llvm/test/Transforms/DCE/intrinsics-systemz.ll
new file mode 100644
index 000000000000000..d54d8d02cbb78d0
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-systemz.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare <16 x i8> @llvm.s390.vlbb(ptr, i32)
+declare <16 x i8> @llvm.s390.vll(i32, ptr)
+declare <16 x i8> @llvm.s390.vlrl(i32, ptr)
+
+define void @test_llvm_s390.vlbb(ptr %a) {
+; CHECK-LABEL: define void @test_llvm_s390.vlbb(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.s390.vlbb(ptr %a, i32 8)
+  ret void
+}
+
+define void @test_llvm_s390.vll(i32 %a, ptr %b) {
+; CHECK-LABEL: define void @test_llvm_s390.vll(
+; CHECK-SAME: i32 [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.s390.vll(i32 %a, ptr %b)
+  ret void
+}
+
+define void @test_llvm_s390.vlrl(i32 %a, ptr %b) {
+; CHECK-LABEL: define void @test_llvm_s390.vlrl(
+; CHECK-SAME: i32 [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.s390.vlrl(i32 %a, ptr %b)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-ve.ll b/llvm/test/Transforms/DCE/intrinsics-ve.ll
new file mode 100644
index 000000000000000..5c6d207587f8e4d
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-ve.ll
@@ -0,0 +1,729 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare i64 @llvm.ve.vl.pack.f32p(ptr, ptr)
+
+define void @test_llvm_ve_vl_pack_f32p(ptr %a, ptr %b) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_pack_f32p(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.ve.vl.pack.f32p(ptr %a, ptr %b)
+  ret void
+}
+
+
+declare i64 @llvm.ve.vl.pack.f32a(ptr)
+
+define void @test_llvm_ve_vl_pack_f32a(ptr %a) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_pack_f32a(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.ve.vl.pack.f32a(ptr %a)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vld_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vld_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldu_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldu_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldunc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldunc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldunc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldunc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldunc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldunc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldunc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldunc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlsx.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldlsx_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlsx_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlsx.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlsx.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldlsx_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlsx_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlsx.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlsxnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldlsxnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlsxnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlsxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlsxnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldlsxnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlsxnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlsxnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlzx.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldlzx_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlzx_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlzx.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlzx.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldlzx_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlzx_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlzx.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlzxnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldlzxnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlzxnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlzxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlzxnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldlzxnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlzxnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlzxnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld2d.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vld2d_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld2d_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld2d.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld2d.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vld2d_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld2d_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld2d.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld2dnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vld2dnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld2dnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld2dnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld2dnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vld2dnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld2dnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld2dnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu2d.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldu2d_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu2d_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu2d.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu2d.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldu2d_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu2d_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu2d.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu2dnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldu2dnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu2dnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu2dnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu2dnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldu2dnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu2dnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu2dnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dsx.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldl2dsx_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dsx_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsx.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dsx.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldl2dsx_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dsx_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsx.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dsxnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldl2dsxnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dsxnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dsxnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldl2dsxnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dsxnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsxnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dzx.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldl2dzx_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dzx_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dzx.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldl2dzx_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dzx_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dzx.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dzxnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldl2dzxnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dzxnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dzxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dzxnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldl2dzxnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dzxnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dzxnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgt_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgt_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgt_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgt_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgt_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgt_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgt_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgt_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtnc_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtnc_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtnc_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtnc_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtnc_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtnc_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtnc_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtnc_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtu_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtu_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtu_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtu_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtu_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtu_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtu_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtu_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtunc_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtunc_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtunc_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtunc_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtunc_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtunc_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtunc_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtunc_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtlsx_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsx_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlsx_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsx_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtlsx_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsx_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlsx_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsx_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtlsxnc_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsxnc_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlsxnc_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsxnc_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtlsxnc_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsxnc_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlsxnc_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsxnc_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtlzx_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzx_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlzx_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzx_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtlzx_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzx_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlzx_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzx_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtlzxnc_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzxnc_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlzxnc_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzxnc_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtlzxnc_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzxnc_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlzxnc_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzxnc_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-wasm.ll b/llvm/test/Transforms/DCE/intrinsics-wasm.ll
new file mode 100644
index 000000000000000..8d6a60965a46d12
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-wasm.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare float @llvm.wasm.loadf16.f32(ptr)
+
+define void @test_llvm_wasm_loadf16_f32(ptr %a) {
+; CHECK-LABEL: define void @test_llvm_wasm_loadf16_f32(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call float @llvm.wasm.loadf16.f32(ptr %a)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/nvvm-ldu-ldg-willreturn.ll b/llvm/test/Transforms/DCE/nvvm-ldu-ldg-willreturn.ll
deleted file mode 100644
index 64a023ef4513768..000000000000000
--- a/llvm/test/Transforms/DCE/nvvm-ldu-ldg-willreturn.ll
+++ /dev/null
@@ -1,187 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S < %s -passes=dce  | FileCheck %s
-
-; ldu/ldg intrinsics were erroneously not marked IntrWillReturn, preventing
-; them from being eliminated at IR level when dead.
-
-declare i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i16 @llvm.nvvm.ldu.global.i.i16.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i32 @llvm.nvvm.ldu.global.i.i32.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i64 @llvm.nvvm.ldu.global.i.i64.p1(ptr addrspace(1) %ptr, i32 %align)
-declare ptr @llvm.nvvm.ldu.global.p.p1(ptr addrspace(1) %ptr, i32 %align)
-declare float @llvm.nvvm.ldu.global.f.f32.p1(ptr addrspace(1) %ptr, i32 %align)
-declare double @llvm.nvvm.ldu.global.f.f64.p1(ptr addrspace(1) %ptr, i32 %align)
-declare half @llvm.nvvm.ldu.global.f.f16.p1(ptr addrspace(1) %ptr, i32 %align)
-declare <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 %align)
-
-declare i8 @llvm.nvvm.ldg.global.i.i8.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i16 @llvm.nvvm.ldg.global.i.i16.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i64 @llvm.nvvm.ldg.global.i.i64.p1(ptr addrspace(1) %ptr, i32 %align)
-declare ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %ptr, i32 %align)
-declare float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %ptr, i32 %align)
-declare double @llvm.nvvm.ldg.global.f.f64.p1(ptr addrspace(1) %ptr, i32 %align)
-declare half @llvm.nvvm.ldg.global.f.f16.p1(ptr addrspace(1) %ptr, i32 %align)
-declare <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 %align)
-
-define void @test_ldu_i8_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_i8_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldu_i16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_i16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i16 @llvm.nvvm.ldu.global.i.i16.p1(ptr addrspace(1) %ptr, i32 2)
-  ret void
-}
-
-define void @test_ldu_i32_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_i32_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i32 @llvm.nvvm.ldu.global.i.i32.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldu_i64_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_i64_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i64 @llvm.nvvm.ldu.global.i.i64.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldu_p_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_p_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call ptr @llvm.nvvm.ldu.global.p.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldu_f32_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_f32_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call float @llvm.nvvm.ldu.global.f.f32.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldu_f64_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_f64_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call double @llvm.nvvm.ldu.global.f.f64.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldu_f16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_f16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call half @llvm.nvvm.ldu.global.f.f16.p1(ptr addrspace(1) %ptr, i32 2)
-  ret void
-}
-
-define void @test_ldu_v2f16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_v2f16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldg_i8_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_i8_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i8 @llvm.nvvm.ldg.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldg_i16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_i16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i16 @llvm.nvvm.ldg.global.i.i16.p1(ptr addrspace(1) %ptr, i32 2)
-  ret void
-}
-
-define void @test_ldg_i32_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_i32_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldg_i64_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_i64_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i64 @llvm.nvvm.ldg.global.i.i64.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldg_p_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_p_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldg_f32_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_f32_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldg_f64_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_f64_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call double @llvm.nvvm.ldg.global.f.f64.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldg_f16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_f16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call half @llvm.nvvm.ldg.global.f.f16.p1(ptr addrspace(1) %ptr, i32 2)
-  ret void
-}
-
-define void @test_ldg_v2f16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_v2f16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
index e6f26aeb98b1412..eb39684a98b5f52 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
@@ -190,6 +190,14 @@ define i32 @atomicrmw_add_global_to_flat_preserve_amdgpu_md(ptr addrspace(1) %gl
   ret i32 %ret
 }
 
+; Make sure there's no assert
+; CHECK-LABEL: @try_infer_getelementptr_constant_null(
+; CHECK-NEXT: %ce = getelementptr i8, ptr getelementptr inbounds (i8, ptr null, i64 8), i64 0
+define ptr @try_infer_getelementptr_constant_null() {
+  %ce = getelementptr i8, ptr getelementptr inbounds (i8, ptr null, i64 8), i64 0
+  ret ptr %ce
+}
+
 attributes #0 = { nounwind }
 
 !0 = !{}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/builtin-assumed-addrspace.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/builtin-assumed-addrspace.ll
index 14614ae3dd6cc12..e0c80c03895417d 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/builtin-assumed-addrspace.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/builtin-assumed-addrspace.ll
@@ -1,84 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=infer-address-spaces -o - %s | FileCheck %s
 
-; CHECK-LABEL: @f0
-; CHECK: addrspacecast ptr {{%.*}} to ptr addrspace(3)
-; CHECK: getelementptr inbounds float, ptr addrspace(3)
-; CHECK: load float, ptr addrspace(3)
 define float @f0(ptr %p) {
+; CHECK-LABEL: define float @f0(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[P]])
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[IS_SHARED]])
+; CHECK-NEXT:    [[WORKITEM_ID_X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[WORKITEM_ID_X]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(3)
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr addrspace(3) [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret float [[LOAD]]
+;
 entry:
-  %0 = call i1 @llvm.amdgcn.is.shared(ptr %p)
-  tail call void @llvm.assume(i1 %0)
-  %1 = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %idxprom = zext i32 %1 to i64
+  %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %p)
+  tail call void @llvm.assume(i1 %is.shared)
+  %workitem.id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idxprom = zext i32 %workitem.id.x to i64
   %arrayidx = getelementptr inbounds float, ptr %p, i64 %idxprom
-  %2 = load float, ptr %arrayidx, align 4
-  ret float %2
+  %load = load float, ptr %arrayidx, align 4
+  ret float %load
 }
 
-; CHECK-LABEL: @f1
-; CHECK: addrspacecast ptr {{%.*}} to ptr addrspace(5)
-; CHECK: getelementptr inbounds float, ptr addrspace(5)
-; CHECK: load float, ptr addrspace(5)
 define float @f1(ptr %p) {
+; CHECK-LABEL: define float @f1(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[P]])
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[IS_PRIVATE]])
+; CHECK-NEXT:    [[WORKITEM_ID_X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[WORKITEM_ID_X]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(5)
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(5) [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr addrspace(5) [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret float [[LOAD]]
+;
 entry:
-  %0 = call i1 @llvm.amdgcn.is.private(ptr %p)
-  tail call void @llvm.assume(i1 %0)
-  %1 = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %idxprom = zext i32 %1 to i64
+  %is.private = call i1 @llvm.amdgcn.is.private(ptr %p)
+  tail call void @llvm.assume(i1 %is.private)
+  %workitem.id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idxprom = zext i32 %workitem.id.x to i64
   %arrayidx = getelementptr inbounds float, ptr %p, i64 %idxprom
-  %2 = load float, ptr %arrayidx, align 4
-  ret float %2
+  %load = load float, ptr %arrayidx, align 4
+  ret float %load
 }
 
-; CHECK-LABEL: @f2
-; CHECK: addrspacecast ptr {{%.*}} to ptr addrspace(1)
-; CHECK: getelementptr inbounds float, ptr addrspace(1)
-; CHECK: load float, ptr addrspace(1)
 define float @f2(ptr %p) {
+; CHECK-LABEL: define float @f2(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[P]])
+; CHECK-NEXT:    [[NOT_PRIVATE:%.*]] = xor i1 [[IS_PRIVATE]], true
+; CHECK-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[P]])
+; CHECK-NEXT:    [[NOT_SHARED:%.*]] = xor i1 [[IS_SHARED]], true
+; CHECK-NEXT:    [[NOT_PRIVATE_AND_NOT_SHARED:%.*]] = and i1 [[NOT_PRIVATE]], [[NOT_SHARED]]
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[NOT_PRIVATE_AND_NOT_SHARED]])
+; CHECK-NEXT:    [[WORKITEM_ID_X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[WORKITEM_ID_X]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(1)
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret float [[LOAD]]
+;
 entry:
-  %0 = call i1 @llvm.amdgcn.is.private(ptr %p)
-  %1 = xor i1 %0, -1
-  %2 = call i1 @llvm.amdgcn.is.shared(ptr %p)
-  %3 = xor i1 %2, -1
-  %4 = and i1 %1, %3
-  tail call void @llvm.assume(i1 %4)
-  %5 = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %idxprom = zext i32 %5 to i64
+  %is.private = call i1 @llvm.amdgcn.is.private(ptr %p)
+  %not.private = xor i1 %is.private, true
+  %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %p)
+  %not.shared = xor i1 %is.shared, true
+  %not.private.and.not.shared = and i1 %not.private, %not.shared
+  tail call void @llvm.assume(i1 %not.private.and.not.shared)
+  %workitem.id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idxprom = zext i32 %workitem.id.x to i64
   %arrayidx = getelementptr inbounds float, ptr %p, i64 %idxprom
-  %6 = load float, ptr %arrayidx, align 4
-  ret float %6
+  %load = load float, ptr %arrayidx, align 4
+  ret float %load
 }
 
-; CHECK-LABEL: @g0
-; CHECK: if.then:
-; CHECK: addrspacecast ptr {{%.*}} to ptr addrspace(3)
-; CHECK: getelementptr inbounds float, ptr addrspace(3)
-; CHECK: load float, ptr addrspace(3)
-; CHECK: if.end:
-; CHECK: getelementptr inbounds float, ptr
-; CHECK: load float, ptr
 define float @g0(i32 %c, ptr %p) {
+; CHECK-LABEL: define float @g0(
+; CHECK-SAME: i32 [[C:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[IF_END:.*]], label %[[IF_THEN:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[P]])
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[IS_SHARED]])
+; CHECK-NEXT:    [[WORKITEM_ID_X_0:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[WORKITEM_ID_X_0]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(3)
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[LOAD0:%.*]] = load float, ptr addrspace(3) [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[LOAD0]], 0.000000e+00
+; CHECK-NEXT:    br label %[[IF_END]]
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    [[S:%.*]] = phi float [ [[ADD]], %[[IF_THEN]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT:    [[WORKITEM_ID_X_1:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = zext i32 [[WORKITEM_ID_X_1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 [[IDXPROM2]]
+; CHECK-NEXT:    [[LOAD1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd float [[S]], [[LOAD1]]
+; CHECK-NEXT:    ret float [[ADD2]]
+;
 entry:
   %tobool.not = icmp eq i32 %c, 0
   br i1 %tobool.not, label %if.end, label %if.then
 
 if.then:
-  %0 = call i1 @llvm.amdgcn.is.shared(ptr %p)
-  tail call void @llvm.assume(i1 %0)
-  %1 = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %idxprom = zext i32 %1 to i64
+  %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %p)
+  tail call void @llvm.assume(i1 %is.shared)
+  %workitem.id.x.0 = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idxprom = zext i32 %workitem.id.x.0 to i64
   %arrayidx = getelementptr inbounds float, ptr %p, i64 %idxprom
-  %2 = load float, ptr %arrayidx, align 4
-  %add = fadd float %2, 0.
+  %load0 = load float, ptr %arrayidx, align 4
+  %add = fadd float %load0, 0.0
   br label %if.end
 
 if.end:
-  %s = phi float [ %add, %if.then ], [ 0., %entry ]
-  %3 = tail call i32 @llvm.amdgcn.workitem.id.y()
-  %idxprom2 = zext i32 %3 to i64
+  %s = phi float [ %add, %if.then ], [ 0.0, %entry ]
+  %workitem.id.x.1 = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %idxprom2 = zext i32 %workitem.id.x.1 to i64
   %arrayidx2 = getelementptr inbounds float, ptr %p, i64 %idxprom2
-  %4 = load float, ptr %arrayidx2, align 4
-  %add2 = fadd float %s, %4
+  %load1 = load float, ptr %arrayidx2, align 4
+  %add2 = fadd float %s, %load1
   ret float %add2
 }
 
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index 3b914dc29ca41c4..bc0d7a509e1f5d2 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -1100,6 +1100,12 @@ declare i64 @write(i32, ptr, i64)
 ; CHECK: declare void @abort() [[NOFREE_COLD:#[0-9]+]]
 declare void @abort()
 
+; CHECK: declare void @__cxa_throw(ptr, ptr, ptr) [[COLD_NORETURN:#[0-9]+]]
+declare void @__cxa_throw(ptr, ptr, ptr)
+
+; CHECK: declare void @_ZSt9terminatev() [[NOFREE_COLD_NORETURN:#[0-9]+]]
+declare void @_ZSt9terminatev()
+
 ; memset_pattern{4,8,16} aren't available everywhere.
 ; CHECK-DARWIN: declare void @memset_pattern4(ptr nocapture writeonly, ptr nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare void @memset_pattern4(ptr, ptr, i64)
@@ -1126,6 +1132,8 @@ declare void @memset_pattern16(ptr, ptr, i64)
 ; CHECK-DAG: attributes [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_REALLOC_ALLOCSIZE1_FAMILY_MALLOC]] = { mustprogress nounwind willreturn allockind("realloc") allocsize(1) memory(argmem: readwrite, inaccessiblemem: readwrite) "alloc-family"="malloc" }
 ; CHECK-DAG: attributes [[INACCESSIBLEMEMORARGONLY_NOFREE_NOUNWIND_WILLRETURN_FAMILY_MALLOC]] = { mustprogress nofree nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) "alloc-family"="malloc" }
 ; CHECK-DAG: attributes [[NOFREE_COLD]] = { cold nofree }
+; CHECK-DAG: attributes [[NOFREE_COLD_NORETURN]] = { cold nofree noreturn }
+; CHECK-DAG: attributes [[COLD_NORETURN]] = { cold noreturn }
 
 ; CHECK-NVPTX-DAG: attributes [[NOFREE_NOUNWIND_READNONE]] = { nofree nosync nounwind memory(none) }
 
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-prf.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-prf.ll
new file mode 100644
index 000000000000000..1de582dff52dad5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-prf.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @test_prf(ptr %base){
+; CHECK-LABEL: define void @test_prf(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> zeroinitializer, ptr %base, i32 1)
+  ret void
+}
+
+define void @test_prfb_gather_index(ptr %base, <vscale x 2 x i64> %indexes){
+; CHECK-LABEL: define void @test_prfb_gather_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfb.gather.index.nx2vi64(<vscale x 2 x i1> zeroinitializer, ptr %base, <vscale x 2 x i64> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfb_gather_scalar_offset(<vscale x 4 x i32> %bases){
+; CHECK-LABEL: define void @test_prfb_gather_scalar_offset(
+; CHECK-SAME: <vscale x 4 x i32> [[BASES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nx4vi32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %bases, i64 7, i32 1)
+  ret void
+}
+
+define void @test_prfb_gather_sxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfb_gather_sxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfb.gather.sxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfb_gather_uxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfb_gather_uxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfb.gather.uxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfd_gather_index(ptr %base, <vscale x 2 x i64> %indexes){
+; CHECK-LABEL: define void @test_prfd_gather_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfd.gather.index.nx2vi64(<vscale x 2 x i1> zeroinitializer, ptr %base, <vscale x 2 x i64> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfd_gather_scalar_offset(<vscale x 4 x i32> %bases){
+; CHECK-LABEL: define void @test_prfd_gather_scalar_offset(
+; CHECK-SAME: <vscale x 4 x i32> [[BASES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfd.gather.scalar.offset.nx4vi32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %bases, i64 7, i32 1)
+  ret void
+}
+
+define void @test_prfd_gather_sxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfd_gather_sxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfd.gather.sxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfd_gather_uxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfd_gather_uxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfd.gather.uxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfh_gather_index(ptr %base, <vscale x 2 x i64> %indexes){
+; CHECK-LABEL: define void @test_prfh_gather_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfh.gather.index.nx2vi64(<vscale x 2 x i1> zeroinitializer, ptr %base, <vscale x 2 x i64> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfh_gather_scalar_offset(<vscale x 4 x i32> %bases){
+; CHECK-LABEL: define void @test_prfh_gather_scalar_offset(
+; CHECK-SAME: <vscale x 4 x i32> [[BASES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfh.gather.scalar.offset.nx4vi32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %bases, i64 7, i32 1)
+  ret void
+}
+
+define void @test_prfh_gather_sxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfh_gather_sxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfh.gather.sxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfh_gather_uxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfh_gather_uxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfh.gather.uxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfw_gather_index(ptr %base, <vscale x 2 x i64> %indexes){
+; CHECK-LABEL: define void @test_prfw_gather_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfw.gather.index.nx2vi64(<vscale x 2 x i1> zeroinitializer, ptr %base, <vscale x 2 x i64> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfw_gather_scalar_offset(<vscale x 4 x i32> %bases){
+; CHECK-LABEL: define void @test_prfw_gather_scalar_offset(
+; CHECK-SAME: <vscale x 4 x i32> [[BASES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfw.gather.scalar.offset.nx4vi32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %bases, i64 7, i32 1)
+  ret void
+}
+
+define void @test_prfw_gather_sxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfw_gather_sxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfw.gather.sxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfw_gather_uxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfw_gather_uxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfw.gather.uxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index adb61cd4b9692fb..b1f21e58de1e2ad 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -99,16 +99,54 @@ define i32 @test3(i32 %A) {
 }
 
 ; D = B + -A = B - A
-define i32 @test4(i32 %A, i32 %B) {
+define i32 @test4(i32 %A, i32 %BB) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B]], [[A:%.*]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
+  %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization
   %C = sub i32 0, %A
   %D = add i32 %B, %C
   ret i32 %D
 }
 
+define i32 @test4_both_nsw(i32 %A, i32 %BB) {
+; CHECK-LABEL: @test4_both_nsw(
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = sub nsw i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization
+  %C = sub nsw i32 0, %A
+  %D = add nsw i32 %B, %C
+  ret i32 %D
+}
+
+define i32 @test4_neg_nsw(i32 %A, i32 %BB) {
+; CHECK-LABEL: @test4_neg_nsw(
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization
+  %C = sub nsw i32 0, %A
+  %D = add i32 %B, %C
+  ret i32 %D
+}
+
+define i32 @test4_add_nsw(i32 %A, i32 %BB) {
+; CHECK-LABEL: @test4_add_nsw(
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization
+  %C = sub i32 0, %A
+  %D = add nsw i32 %B, %C
+  ret i32 %D
+}
+
 ; D = -A + B = B - A
 define i32 @test5(i32 %A, i32 %B) {
 ; CHECK-LABEL: @test5(
diff --git a/llvm/test/Transforms/InstCombine/and-or-icmp-const-icmp.ll b/llvm/test/Transforms/InstCombine/and-or-icmp-const-icmp.ll
index 9143a2d0ccda249..9365f8281ccbd3b 100644
--- a/llvm/test/Transforms/InstCombine/and-or-icmp-const-icmp.ll
+++ b/llvm/test/Transforms/InstCombine/and-or-icmp-const-icmp.ll
@@ -13,8 +13,8 @@
 ; Basic tests
 ; ==============================================================================
 define i1 @eq_basic(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @eq_basic
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @eq_basic(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[OR]]
@@ -26,8 +26,8 @@ define i1 @eq_basic(i8 %x, i8 %y) {
 }
 
 define i1 @ne_basic_equal_5(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @ne_basic_equal_5
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @ne_basic_equal_5(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -6
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[AND]]
@@ -40,8 +40,8 @@ define i1 @ne_basic_equal_5(i8 %x, i8 %y) {
 }
 
 define i1 @eq_basic_equal_minus_1(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @eq_basic_equal_minus_1
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @eq_basic_equal_minus_1(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge i8 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[OR]]
 ;
@@ -53,8 +53,8 @@ define i1 @eq_basic_equal_minus_1(i8 %x, i8 %y) {
 }
 
 define i1 @ne_basic_equal_minus_7(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @ne_basic_equal_minus_7
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @ne_basic_equal_minus_7(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], 6
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[AND]]
@@ -67,8 +67,8 @@ define i1 @ne_basic_equal_minus_7(i8 %x, i8 %y) {
 }
 
 define i1 @eq_basic_unequal(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @eq_basic_unequal
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @eq_basic_unequal(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[SUB:%.*]] = add i8 [[X]], -5
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[X]], 6
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i8 [[SUB]], [[Y]]
@@ -83,8 +83,8 @@ define i1 @eq_basic_unequal(i8 %x, i8 %y) {
 }
 
 define i1 @ne_basic_unequal(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @ne_basic_unequal
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @ne_basic_unequal(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X]], 7
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ne i8 [[X]], -4
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ule i8 [[ADD]], [[Y]]
@@ -102,8 +102,8 @@ define i1 @ne_basic_unequal(i8 %x, i8 %y) {
 ; Tests with multiple uses
 ; ==============================================================================
 define i1 @eq_multi_c1(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @eq_multi_c1
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @eq_multi_c1(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[X]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge i8 [[TMP1]], [[Y]]
@@ -118,8 +118,8 @@ define i1 @eq_multi_c1(i8 %x, i8 %y) {
 }
 
 define i1 @ne_multi_c2(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @ne_multi_c2
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @ne_multi_c2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ule i8 [[X]], [[Y]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -1
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult i8 [[TMP1]], [[Y]]
@@ -137,8 +137,8 @@ define i1 @ne_multi_c2(i8 %x, i8 %y) {
 ; Tests with vector types
 ; ==============================================================================
 define <2 x i1> @eq_vector(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 -1, i8 -1>
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[OR]]
@@ -150,8 +150,8 @@ define <2 x i1> @eq_vector(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @ne_vector_equal_5(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @ne_vector_equal_5
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @ne_vector_equal_5(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 -6, i8 -6>
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[AND]]
@@ -164,8 +164,8 @@ define <2 x i1> @ne_vector_equal_5(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @eq_vector_equal_minus_1(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector_equal_minus_1
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector_equal_minus_1(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge <2 x i8> [[X]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[OR]]
 ;
@@ -177,8 +177,8 @@ define <2 x i1> @eq_vector_equal_minus_1(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @ne_vector_equal_minus_7(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @ne_vector_equal_minus_7
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @ne_vector_equal_minus_7(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 6, i8 6>
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[AND]]
@@ -191,8 +191,8 @@ define <2 x i1> @ne_vector_equal_minus_7(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @eq_vector_unequal1(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector_unequal1
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector_unequal1(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i8> [[X]], <i8 -5, i8 -5>
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq <2 x i8> [[X]], <i8 2, i8 2>
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ugt <2 x i8> [[SUB]], [[Y]]
@@ -207,8 +207,8 @@ define <2 x i1> @eq_vector_unequal1(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @ne_vector_unequal2(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @ne_vector_unequal2
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @ne_vector_unequal2(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i8> [[X]], <i8 7, i8 7>
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ne <2 x i8> [[X]], <i8 -3, i8 -3>
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ule <2 x i8> [[ADD]], [[Y]]
@@ -226,8 +226,8 @@ define <2 x i1> @ne_vector_unequal2(<2 x i8> %x, <2 x i8> %y) {
 ; Tests with poison
 ; ==============================================================================
 define <2 x i1> @eq_vector_poison_icmp(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector_poison_icmp
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector_poison_icmp(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 -6, i8 -6>
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[OR]]
@@ -240,8 +240,8 @@ define <2 x i1> @eq_vector_poison_icmp(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @eq_vector_poison_add(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector_poison_add
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector_poison_add(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 -6, i8 -6>
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[OR]]
@@ -257,8 +257,8 @@ define <2 x i1> @eq_vector_poison_add(<2 x i8> %x, <2 x i8> %y) {
 ; Tests with values commuted
 ; ==============================================================================
 define i1 @eq_commuted(i8 %x, i8 %py) {
-; CHECK-LABEL: define i1 @eq_commuted
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[PY:%.*]]) {
+; CHECK-LABEL: define i1 @eq_commuted(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[PY:%.*]]) {
 ; CHECK-NEXT:    [[Y:%.*]] = sdiv i8 43, [[PY]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge i8 [[TMP1]], [[Y]]
@@ -272,8 +272,8 @@ define i1 @eq_commuted(i8 %x, i8 %py) {
 }
 
 define i1 @ne_commuted_equal_minus_1(i8 %x, i8 %py) {
-; CHECK-LABEL: define i1 @ne_commuted_equal_minus_1
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[PY:%.*]]) {
+; CHECK-LABEL: define i1 @ne_commuted_equal_minus_1(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[PY:%.*]]) {
 ; CHECK-NEXT:    [[Y:%.*]] = sdiv i8 42, [[PY]]
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ugt i8 [[Y]], [[X]]
 ; CHECK-NEXT:    ret i1 [[AND]]
diff --git a/llvm/test/Transforms/InstCombine/and-xor-merge.ll b/llvm/test/Transforms/InstCombine/and-xor-merge.ll
index e6df4e32bae361c..80bdf67525faaea 100644
--- a/llvm/test/Transforms/InstCombine/and-xor-merge.ll
+++ b/llvm/test/Transforms/InstCombine/and-xor-merge.ll
@@ -43,8 +43,8 @@ define i32 @PR38781(i32 %a, i32 %b) {
 
 ; (a ^ 4) & (a ^ ~4) -> 0
 define i32 @PR75692_1(i32 %x) {
-; CHECK-LABEL: @PR75692_1
-; CHECK-NEXT:  ret i32 0
+; CHECK-LABEL: @PR75692_1(
+; CHECK-NEXT:    ret i32 0
 ;
   %t2 = xor i32 %x, 4
   %t3 = xor i32 %x, -5
@@ -54,11 +54,11 @@ define i32 @PR75692_1(i32 %x) {
 
 ; (a ^ 4) & (a ^ 3) is not zero
 define i32 @PR75692_2(i32 %x) {
-; CHECK-LABEL: @PR75692_2
-; CHECK-NEXT:  %t2 = xor i32 %x, 4
-; CHECK-NEXT:  %t3 = xor i32 %x, -4
-; CHECK-NEXT:  %t4 = and i32 %t2, %t3
-; CHECK-NEXT:  ret i32 %t4
+; CHECK-LABEL: @PR75692_2(
+; CHECK-NEXT:    [[T2:%.*]] = xor i32 [[X:%.*]], 4
+; CHECK-NEXT:    [[T3:%.*]] = xor i32 [[X]], -4
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T2]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t2 = xor i32 %x, 4
   %t3 = xor i32 %x, -4
@@ -68,11 +68,11 @@ define i32 @PR75692_2(i32 %x) {
 
 ; (a ^ 4) & (b ^ ~4) is not zero, since a != b is possible
 define i32 @PR75692_3(i32 %x, i32 %y) {
-; CHECK-LABEL: @PR75692_3
-; CHECK-NEXT:  %t2 = xor i32 %x, 4
-; CHECK-NEXT:  %t3 = xor i32 %y, -5
-; CHECK-NEXT:  %t4 = and i32 %t2, %t3
-; CHECK-NEXT:  ret i32 %t4
+; CHECK-LABEL: @PR75692_3(
+; CHECK-NEXT:    [[T2:%.*]] = xor i32 [[X:%.*]], 4
+; CHECK-NEXT:    [[T3:%.*]] = xor i32 [[Y:%.*]], -5
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T2]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t2 = xor i32 %x, 4
   %t3 = xor i32 %y, -5
diff --git a/llvm/test/Transforms/InstCombine/ashr-lshr.ll b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
index c2a4f35412670be..a81cd47b1cd4b64 100644
--- a/llvm/test/Transforms/InstCombine/ashr-lshr.ll
+++ b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
@@ -61,6 +61,18 @@ define i32 @ashr_lshr2(i32 %x, i32 %y) {
   ret i32 %ret
 }
 
+define i128 @ashr_lshr2_i128(i128 %x, i128 %y) {
+; CHECK-LABEL: @ashr_lshr2_i128(
+; CHECK-NEXT:    [[CMP1:%.*]] = ashr i128 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i128 [[CMP1]]
+;
+  %cmp = icmp sgt i128 %x, 5
+  %l = lshr i128 %x, %y
+  %r = ashr exact i128 %x, %y
+  %ret = select i1 %cmp, i128 %l, i128 %r
+  ret i128 %ret
+}
+
 define <2 x i32> @ashr_lshr_splat_vec(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @ashr_lshr_splat_vec(
 ; CHECK-NEXT:    [[CMP12:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
index 0a7de501ca0225f..443efbe1ecaf6d9 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
@@ -16,8 +16,8 @@
 
 define i1 @p0(i8 %x, i8 %y) {
 ; CHECK-LABEL: @p0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -33,8 +33,8 @@ define i1 @p0(i8 %x, i8 %y) {
 
 define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @p1_vec(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq <2 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <2 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
 ;
   %t0 = shl <2 x i8> <i8 -1, i8 -1>, %y
@@ -46,8 +46,8 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
 
 define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p2_vec_poison0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 poison, i8 -1>, %y
@@ -59,8 +59,8 @@ define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p3_vec_poison0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 -1, i8 -1>, %y
@@ -72,8 +72,8 @@ define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i1> @p4_vec_poison2(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p4_vec_poison2(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 poison, i8 -1>, %y
@@ -92,8 +92,8 @@ declare i8 @gen8()
 define i1 @c0(i8 %y) {
 ; CHECK-LABEL: @c0(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -107,8 +107,8 @@ define i1 @c0(i8 %y) {
 define i1 @c1(i8 %y) {
 ; CHECK-LABEL: @c1(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -122,8 +122,8 @@ define i1 @c1(i8 %y) {
 define i1 @c2(i8 %y) {
 ; CHECK-LABEL: @c2(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -144,8 +144,8 @@ define i1 @oneuse0(i8 %x, i8 %y) {
 ; CHECK-LABEL: @oneuse0(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = and i8 [[T0]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -285,9 +285,9 @@ define i1 @n1(i8 %x, i8 %y) {
 define i1 @n2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n2(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -2
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[T0]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
index 54ff87676e71d9c..ffde4eae777cb28 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
@@ -16,8 +16,8 @@
 
 define i1 @p0(i8 %x, i8 %y) {
 ; CHECK-LABEL: @p0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -33,8 +33,8 @@ define i1 @p0(i8 %x, i8 %y) {
 
 define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @p1_vec(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
 ;
   %t0 = shl <2 x i8> <i8 -1, i8 -1>, %y
@@ -46,8 +46,8 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
 
 define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p2_vec_poison0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 poison, i8 -1>, %y
@@ -59,8 +59,8 @@ define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p3_vec_poison0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 -1, i8 -1>, %y
@@ -72,8 +72,8 @@ define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i1> @p4_vec_poison2(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p4_vec_poison2(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 poison, i8 -1>, %y
@@ -92,8 +92,8 @@ declare i8 @gen8()
 define i1 @c0(i8 %y) {
 ; CHECK-LABEL: @c0(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -107,8 +107,8 @@ define i1 @c0(i8 %y) {
 define i1 @c1(i8 %y) {
 ; CHECK-LABEL: @c1(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -122,8 +122,8 @@ define i1 @c1(i8 %y) {
 define i1 @c2(i8 %y) {
 ; CHECK-LABEL: @c2(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -144,8 +144,8 @@ define i1 @oneuse0(i8 %x, i8 %y) {
 ; CHECK-LABEL: @oneuse0(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = and i8 [[T0]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -285,9 +285,9 @@ define i1 @n1(i8 %x, i8 %y) {
 define i1 @n2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n2(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -2
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[T0]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
index c7c57b601eab384..946bb03e04f7e4e 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
@@ -251,9 +251,9 @@ define i1 @n1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n1(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[T0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y ; not 1
@@ -268,9 +268,9 @@ define i1 @n2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n2(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nuw i8 1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub nuw i8 -2, [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw i8 -2, [[T0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 1, %y
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
index d5826524f1637c2..63d406d54179fc3 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
@@ -251,9 +251,9 @@ define i1 @n1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n1(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[T0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y ; not 1
@@ -268,9 +268,9 @@ define i1 @n2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n2(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nuw i8 1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub nuw i8 -2, [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw i8 -2, [[T0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 1, %y
diff --git a/llvm/test/Transforms/InstCombine/cast.ll b/llvm/test/Transforms/InstCombine/cast.ll
index 564e82945c13f03..43c198b9e19d8eb 100644
--- a/llvm/test/Transforms/InstCombine/cast.ll
+++ b/llvm/test/Transforms/InstCombine/cast.ll
@@ -512,8 +512,8 @@ define <2 x i16> @test40vec_poison(<2 x i16> %a) {
 ; ALL-LABEL: @test40vec_poison(
 ; ALL-NEXT:    [[T21:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 9, i16 poison>
 ; ALL-NEXT:    [[T5:%.*]] = shl <2 x i16> [[A]], <i16 8, i16 poison>
-; ALL-NEXT:    [[R:%.*]] = or disjoint <2 x i16> [[T21]], [[T5]]
-; ALL-NEXT:    ret <2 x i16> [[R]]
+; ALL-NEXT:    [[T32:%.*]] = or disjoint <2 x i16> [[T21]], [[T5]]
+; ALL-NEXT:    ret <2 x i16> [[T32]]
 ;
   %t = zext <2 x i16> %a to <2 x i32>
   %t21 = lshr <2 x i32> %t, <i32 9, i32 poison>
@@ -1994,8 +1994,8 @@ define <2 x i8> @trunc_lshr_zext_uniform(<2 x i8> %A) {
 
 define <2 x i8> @trunc_lshr_zext_uniform_poison(<2 x i8> %A) {
 ; ALL-LABEL: @trunc_lshr_zext_uniform_poison(
-; ALL-NEXT:    [[D:%.*]] = lshr <2 x i8> [[A:%.*]], <i8 6, i8 poison>
-; ALL-NEXT:    ret <2 x i8> [[D]]
+; ALL-NEXT:    [[C:%.*]] = lshr <2 x i8> [[A:%.*]], <i8 6, i8 poison>
+; ALL-NEXT:    ret <2 x i8> [[C]]
 ;
   %B = zext <2 x i8> %A to <2 x i32>
   %C = lshr <2 x i32> %B, <i32 6, i32 poison>
@@ -2016,8 +2016,8 @@ define <2 x i8> @trunc_lshr_zext_nonuniform(<2 x i8> %A) {
 
 define <3 x i8> @trunc_lshr_zext_nonuniform_poison(<3 x i8> %A) {
 ; ALL-LABEL: @trunc_lshr_zext_nonuniform_poison(
-; ALL-NEXT:    [[D:%.*]] = lshr <3 x i8> [[A:%.*]], <i8 6, i8 2, i8 poison>
-; ALL-NEXT:    ret <3 x i8> [[D]]
+; ALL-NEXT:    [[C:%.*]] = lshr <3 x i8> [[A:%.*]], <i8 6, i8 2, i8 poison>
+; ALL-NEXT:    ret <3 x i8> [[C]]
 ;
   %B = zext <3 x i8> %A to <3 x i32>
   %C = lshr <3 x i32> %B, <i32 6, i32 2, i32 poison>
diff --git a/llvm/test/Transforms/InstCombine/crash.ll b/llvm/test/Transforms/InstCombine/crash.ll
index 5f86069ef736803..9b37d6943b9e52a 100644
--- a/llvm/test/Transforms/InstCombine/crash.ll
+++ b/llvm/test/Transforms/InstCombine/crash.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=instcombine -S
-target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128:n8:16:32"
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin10.0"
 
 define i32 @test0(i8 %tmp2) ssp {
diff --git a/llvm/test/Transforms/InstCombine/demorgan.ll b/llvm/test/Transforms/InstCombine/demorgan.ll
index c9196b6d49afffd..11052d38f9bc7e1 100644
--- a/llvm/test/Transforms/InstCombine/demorgan.ll
+++ b/llvm/test/Transforms/InstCombine/demorgan.ll
@@ -119,8 +119,8 @@ define i32 @test3(i32 %A, i32 %B) {
 
 define i32 @test4(i32 %A) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[NOTC1:%.*]] = or i32 [[A:%.*]], -6
-; CHECK-NEXT:    ret i32 [[NOTC1]]
+; CHECK-NEXT:    [[NOTC:%.*]] = or i32 [[A:%.*]], -6
+; CHECK-NEXT:    ret i32 [[NOTC]]
 ;
   %nota = xor i32 %A, -1
   %c = and i32 %nota, 5
diff --git a/llvm/test/Transforms/InstCombine/fast-math.ll b/llvm/test/Transforms/InstCombine/fast-math.ll
index da403555ebe2489..d7c90e82ab520a8 100644
--- a/llvm/test/Transforms/InstCombine/fast-math.ll
+++ b/llvm/test/Transforms/InstCombine/fast-math.ll
@@ -709,13 +709,15 @@ define double @sqrt_intrinsic_three_args5(double %x, double %y) {
   ret double %sqrt
 }
 
-define double @sqrt_intrinsic_three_args6(double %x, double %y) {
+define double @sqrt_intrinsic_three_args6(double %x, ptr %yp) {
 ; CHECK-LABEL: @sqrt_intrinsic_three_args6(
+; CHECK-NEXT:    [[Y:%.*]] = load double, ptr [[YP:%.*]], align 8
 ; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y]])
 ; CHECK-NEXT:    [[SQRT:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
 ; CHECK-NEXT:    ret double [[SQRT]]
 ;
+  %y = load double, ptr %yp ; thwart complexity-based canonicalization
   %mul = fmul fast double %x, %x
   %mul2 = fmul fast double %y, %mul
   %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
diff --git a/llvm/test/Transforms/InstCombine/icmp-and-shift.ll b/llvm/test/Transforms/InstCombine/icmp-and-shift.ll
index 65b9749ffc1bf52..2f797d726afe322 100644
--- a/llvm/test/Transforms/InstCombine/icmp-and-shift.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-and-shift.ll
@@ -522,8 +522,8 @@ define i1 @slt_and_shl_one(i8 %x, i8 %y) {
 
 define i1 @fold_eq_lhs(i8 %x, i8 %y) {
 ; CHECK-LABEL: @fold_eq_lhs(
-; CHECK-NEXT:    [[AND:%.*]] = lshr i8 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %shl = shl i8 -1, %x
@@ -563,8 +563,8 @@ define i1 @fold_eq_lhs_fail_multiuse_shl(i8 %x, i8 %y) {
 define i1 @fold_ne_rhs(i8 %x, i8 %yy) {
 ; CHECK-LABEL: @fold_ne_rhs(
 ; CHECK-NEXT:    [[Y:%.*]] = xor i8 [[YY:%.*]], 123
-; CHECK-NEXT:    [[AND:%.*]] = lshr i8 [[Y]], [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %y = xor i8 %yy, 123
@@ -622,8 +622,8 @@ define i1 @test_shr_and_1_ne_0(i32 %a, i32 %b) {
 define i1 @test_const_shr_and_1_ne_0(i32 %b) {
 ; CHECK-LABEL: @test_const_shr_and_1_ne_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 1, [[B:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP1]], 42
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shr = lshr i32 42, %b
@@ -635,9 +635,9 @@ define i1 @test_const_shr_and_1_ne_0(i32 %b) {
 define i1 @test_not_const_shr_and_1_ne_0(i32 %b) {
 ; CHECK-LABEL: @test_not_const_shr_and_1_ne_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 1, [[B:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP1]], 42
-; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
-; CHECK-NEXT:    ret i1 [[CMP_NOT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shr = lshr i32 42, %b
   %and = and i32 %shr, 1
@@ -648,8 +648,8 @@ define i1 @test_not_const_shr_and_1_ne_0(i32 %b) {
 define i1 @test_const_shr_exact_and_1_ne_0(i32 %b) {
 ; CHECK-LABEL: @test_const_shr_exact_and_1_ne_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 1, [[B:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP1]], 42
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shr = lshr exact i32 42, %b
diff --git a/llvm/test/Transforms/InstCombine/lshr-ashr-of-uscmp.ll b/llvm/test/Transforms/InstCombine/lshr-ashr-of-uscmp.ll
new file mode 100644
index 000000000000000..93082de93f97a4c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lshr-ashr-of-uscmp.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+declare void @use(i8 %val)
+
+; ucmp/scmp(x, y) >> N folds to either zext(x < y) or sext(x < y)
+; if N is one less than the width of result of ucmp/scmp
+define i8 @ucmp_to_zext(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_to_zext(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = lshr i8 %1, 7
+  ret i8 %2
+}
+
+define i8 @ucmp_to_sext(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_to_sext(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = ashr i8 %1, 7
+  ret i8 %2
+}
+
+define i8 @scmp_to_zext(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_to_zext(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  %2 = lshr i8 %1, 7
+  ret i8 %2
+}
+
+define i8 @scmp_to_sext(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_to_sext(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  %2 = ashr i8 %1, 7
+  ret i8 %2
+}
+
+define <4 x i8> @scmp_to_sext_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: define <4 x i8> @scmp_to_sext_vec(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i8>
+; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
+;
+  %1 = call <4 x i8> @llvm.scmp(<4 x i32> %x, <4 x i32> %y)
+  %2 = ashr <4 x i8> %1, <i8 7, i8 7, i8 7, i8 7>
+  ret <4 x i8> %2
+}
+
+; Negative test: incorrect shift amount
+define i8 @ucmp_to_zext_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_to_zext_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i8 [[TMP1]], 5
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = lshr i8 %1, 5
+  ret i8 %2
+}
+
+; Negative test: shift amount is not a constant
+define i8 @ucmp_to_zext_neg2(i32 %x, i32 %y, i8 %s) {
+; CHECK-LABEL: define i8 @ucmp_to_zext_neg2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i8 [[S:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i8 [[TMP1]], [[S]]
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = lshr i8 %1, %s
+  ret i8 %2
+}
+
+; Negative test: the result of ucmp/scmp is used more than once
+define i8 @ucmp_to_zext_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_to_zext_neg3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    call void @use(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i8 [[TMP1]], 7
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  call void @use(i8 %1)
+  %2 = lshr i8 %1, 7
+  ret i8 %2
+}
diff --git a/llvm/test/Transforms/InstCombine/phi.ll b/llvm/test/Transforms/InstCombine/phi.ll
index b12982dd27e404d..2673b1d74bb6fb8 100644
--- a/llvm/test/Transforms/InstCombine/phi.ll
+++ b/llvm/test/Transforms/InstCombine/phi.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128:n8:16:32:64"
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define i32 @test1(i32 %A, i1 %b) {
 ; CHECK-LABEL: @test1(
diff --git a/llvm/test/Transforms/InstCombine/pr96012.ll b/llvm/test/Transforms/InstCombine/pr96012.ll
new file mode 100644
index 000000000000000..1a1356f07421609
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr96012.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+
+; Make sure we don't hang here.
+define i32 @pr96012() {
+; CHECK-LABEL: define i32 @pr96012() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[FOR_COND:.*]]
+; CHECK:       [[FOR_COND]]:
+; CHECK-NEXT:    br label %[[FOR_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond:
+  %indvar = phi i32 [ 1, %entry ], [ %shl, %for.cond ]
+  %conv1 = trunc i32 %indvar to i8
+  %neg = sub i8 0, %conv1
+  %conv2 = zext i8 %neg to i32
+  %shl = shl nuw i32 %conv2, 24
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/InstCombine/scmp.ll b/llvm/test/Transforms/InstCombine/scmp.ll
index 2523872562cad98..7f374c5f9a1d645 100644
--- a/llvm/test/Transforms/InstCombine/scmp.ll
+++ b/llvm/test/Transforms/InstCombine/scmp.ll
@@ -157,6 +157,31 @@ define i1 @scmp_sle_neg_1(i32 %x, i32 %y) {
   ret i1 %2
 }
 
+; scmp(x, y) u< C => x s>= y when C u> 1 and C != -1
+define i1 @scmp_ult_positive_const_gt_than_1_lt_than_umax(i32 %x, i32 %y) {
+; CHECK-LABEL: define i1 @scmp_ult_positive_const_gt_than_1_lt_than_umax(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sge i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  %2 = icmp ult i8 %1, 4
+  ret i1 %2
+}
+
+; scmp(x, y) s> C => x s< y when C != 0 and C != -1
+define i1 @ucmp_ugt_const_not_0_or_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i1 @ucmp_ugt_const_not_0_or_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  %2 = icmp ugt i8 %1, 12
+  ret i1 %2
+}
+
+
 ; ========== Fold -scmp(x, y) => scmp(y, x) ==========
 define i8 @scmp_negated(i32 %x, i32 %y) {
 ; CHECK-LABEL: define i8 @scmp_negated(
@@ -183,3 +208,59 @@ define i8 @scmp_negated_multiuse(i32 %x, i32 %y) {
   %2 = sub i8 0, %1
   ret i8 %2
 }
+
+; Fold ((x s< y) ? -1 : (x != y)) into scmp(x, y)
+define i8 @scmp_from_select_lt(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_from_select_lt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp slt i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Vector version
+define <4 x i8> @scmp_from_select_vec_lt(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: define <4 x i8> @scmp_from_select_vec_lt(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i8> @llvm.scmp.v4i8.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %ne_bool = icmp ne <4 x i32> %x, %y
+  %ne = zext <4 x i1> %ne_bool to <4 x i8>
+  %lt = icmp slt <4 x i32> %x, %y
+  %r = select <4 x i1> %lt, <4 x i8> splat(i8 -1), <4 x i8> %ne
+  ret <4 x i8> %r
+}
+
+; Fold (x s<= y) ? sext(x != y) : 1 into scmp(x, y)
+define i8 @scmp_from_select_le(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_from_select_le(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp sle i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Fold (x s>= y) ? zext(x != y) : -1 into scmp(x, y)
+define i8 @scmp_from_select_ge(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_from_select_ge(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp sge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/select-select.ll b/llvm/test/Transforms/InstCombine/select-select.ll
index 84fe973093e327f..5460ba1bc558384 100644
--- a/llvm/test/Transforms/InstCombine/select-select.ll
+++ b/llvm/test/Transforms/InstCombine/select-select.ll
@@ -282,10 +282,7 @@ define i8 @strong_order_cmp_ugt_eq(i32 %a, i32 %b) {
 
 define i8 @strong_order_cmp_eq_slt(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_slt(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = zext i1 [[CMP_EQ]] to i8
-; CHECK-NEXT:    [[CMP_LT:%.*]] = icmp slt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_LT:%.*]] = select i1 [[CMP_LT]], i8 -1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_LT:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i8 [[SEL_LT]]
 ;
   %cmp.eq = icmp eq i32 %a, %b
@@ -297,10 +294,7 @@ define i8 @strong_order_cmp_eq_slt(i32 %a, i32 %b) {
 
 define i8 @strong_order_cmp_eq_sgt(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_sgt(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext i1 [[CMP_EQ]] to i8
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp sgt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select i1 [[CMP_GT]], i8 1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i8 [[SEL_GT]]
 ;
   %cmp.eq = icmp eq i32 %a, %b
@@ -312,10 +306,7 @@ define i8 @strong_order_cmp_eq_sgt(i32 %a, i32 %b) {
 
 define i8 @strong_order_cmp_eq_ult(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ult(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = zext i1 [[CMP_EQ]] to i8
-; CHECK-NEXT:    [[CMP_LT:%.*]] = icmp ult i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_LT:%.*]] = select i1 [[CMP_LT]], i8 -1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_LT:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i8 [[SEL_LT]]
 ;
   %cmp.eq = icmp eq i32 %a, %b
@@ -327,10 +318,7 @@ define i8 @strong_order_cmp_eq_ult(i32 %a, i32 %b) {
 
 define i8 @strong_order_cmp_eq_ugt(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext i1 [[CMP_EQ]] to i8
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select i1 [[CMP_GT]], i8 1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i8 [[SEL_GT]]
 ;
   %cmp.eq = icmp eq i32 %a, %b
@@ -404,9 +392,7 @@ define i8 @strong_order_cmp_ne_ugt_ne_not_one_use(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_ne_ugt_ne_not_one_use(
 ; CHECK-NEXT:    [[CMP_NE:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    call void @use1(i1 [[CMP_NE]])
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext i1 [[CMP_NE]] to i8
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select i1 [[CMP_GT]], i8 1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[A]], i32 [[B]])
 ; CHECK-NEXT:    ret i8 [[SEL_GT]]
 ;
   %cmp.ne = icmp ne i32 %a, %b
@@ -535,10 +521,7 @@ define <2 x i8> @strong_order_cmp_ugt_ult_vector_poison(<2 x i32> %a, <2 x i32>
 
 define <2 x i8> @strong_order_cmp_eq_ugt_vector(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt_vector(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext <2 x i1> [[CMP_EQ]] to <2 x i8>
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt <2 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select <2 x i1> [[CMP_GT]], <2 x i8> <i8 1, i8 1>, <2 x i8> [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call <2 x i8> @llvm.ucmp.v2i8.v2i32(<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x i8> [[SEL_GT]]
 ;
   %cmp.eq = icmp eq <2 x i32> %a, %b
@@ -550,10 +533,7 @@ define <2 x i8> @strong_order_cmp_eq_ugt_vector(<2 x i32> %a, <2 x i32> %b) {
 
 define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison1(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt_vector_poison1(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext <2 x i1> [[CMP_EQ]] to <2 x i8>
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt <2 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select <2 x i1> [[CMP_GT]], <2 x i8> <i8 1, i8 1>, <2 x i8> [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call <2 x i8> @llvm.ucmp.v2i8.v2i32(<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x i8> [[SEL_GT]]
 ;
   %cmp.eq = icmp eq <2 x i32> %a, %b
@@ -565,10 +545,7 @@ define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison1(<2 x i32> %a, <2 x i32>
 
 define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison2(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt_vector_poison2(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext <2 x i1> [[CMP_EQ]] to <2 x i8>
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt <2 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select <2 x i1> [[CMP_GT]], <2 x i8> <i8 1, i8 1>, <2 x i8> [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call <2 x i8> @llvm.ucmp.v2i8.v2i32(<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x i8> [[SEL_GT]]
 ;
   %cmp.eq = icmp eq <2 x i32> %a, %b
@@ -580,10 +557,7 @@ define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison2(<2 x i32> %a, <2 x i32>
 
 define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison3(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt_vector_poison3(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext <2 x i1> [[CMP_EQ]] to <2 x i8>
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt <2 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select <2 x i1> [[CMP_GT]], <2 x i8> <i8 1, i8 poison>, <2 x i8> [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call <2 x i8> @llvm.ucmp.v2i8.v2i32(<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x i8> [[SEL_GT]]
 ;
   %cmp.eq = icmp eq <2 x i32> %a, %b
diff --git a/llvm/test/Transforms/InstCombine/shl-factor.ll b/llvm/test/Transforms/InstCombine/shl-factor.ll
index 38eecaeff8e41e3..996b15f27f6d345 100644
--- a/llvm/test/Transforms/InstCombine/shl-factor.ll
+++ b/llvm/test/Transforms/InstCombine/shl-factor.ll
@@ -265,3 +265,27 @@ define i6 @sub_shl_same_amount_partial_nuw2(i6 %x, i6 %y, i6 %z) {
   ret i6 %diff
 }
 
+define i8 @add_shl_same_amount_constants(i8 %z) {
+; CHECK-LABEL: @add_shl_same_amount_constants(
+; CHECK-NEXT:    [[SUM:%.*]] = shl i8 7, [[Z:%.*]]
+; CHECK-NEXT:    ret i8 [[SUM]]
+;
+  %s1 = shl i8 4, %z
+  %s2 = shl i8 3, %z
+  %sum = add i8 %s1, %s2
+  ret i8 %sum
+}
+
+define i8 @add_shl_same_amount_constants_extra_use(i8 %z) {
+; CHECK-LABEL: @add_shl_same_amount_constants_extra_use(
+; CHECK-NEXT:    [[S1:%.*]] = shl i8 4, [[Z:%.*]]
+; CHECK-NEXT:    [[SUM:%.*]] = shl i8 7, [[Z]]
+; CHECK-NEXT:    call void @use8(i8 [[S1]])
+; CHECK-NEXT:    ret i8 [[SUM]]
+;
+  %s1 = shl i8 4, %z
+  %s2 = shl i8 3, %z
+  %sum = add i8 %s1, %s2
+  call void @use8(i8 %s1)
+  ret i8 %sum
+}
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
index ecfafbc69797b15..83471f1d0f613fc 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
@@ -340,6 +340,94 @@ define void @array_new_align_nothrow_hot_cold() {
   ret void
 }
 
+;; Check that operator __size_returning_new(unsigned long) converted to
+;; __size_returning_new(unsigned long, __hot_cold_t) with a hot or cold attribute.
+; HOTCOLD-LABEL: @size_returning_test()
+define void @size_returning_test() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[COLD]])
+  %call = call {ptr, i64} @__size_returning_new(i64 10) #3
+  %p  = extractvalue {ptr, i64} %call, 0
+  call void @dummy(ptr %p)
+  ;; Attribute notcold has no effect.
+  ; HOTCOLD: @__size_returning_new(i64 10)
+  %call1 = call {ptr, i64} @__size_returning_new(i64 10) #4
+  %p1  = extractvalue {ptr, i64} %call1, 0
+  call void @dummy(ptr %p1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[HOT]])
+  %call2 = call {ptr, i64} @__size_returning_new(i64 10) #5
+  %p2  = extractvalue {ptr, i64} %call2, 0
+  call void @dummy(ptr %p2)
+  ret void
+}
+
+;; Check that operator __size_returning_new_aligned(unsigned long, std::align_val_t) converted to
+;; __size_returning_new_aligned(unsigned long, std::align_val_t, __hot_cold_t) with a hot or cold attribute.
+; HOTCOLD-LABEL: @size_returning_aligned_test()
+define void @size_returning_aligned_test() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[COLD]])
+  %call = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #3
+  %p  = extractvalue {ptr, i64} %call, 0
+  call void @dummy(ptr %p)
+  ;; Attribute notcold has no effect.
+  ; HOTCOLD: @__size_returning_new_aligned(i64 10, i64 8)
+  %call1 = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #4
+  %p1  = extractvalue {ptr, i64} %call1, 0
+  call void @dummy(ptr %p1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[HOT]])
+  %call2 = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #5
+  %p2  = extractvalue {ptr, i64} %call2, 0
+  call void @dummy(ptr %p2)
+  ret void
+}
+
+;; Check that __size_returning_new_hot_cold(unsigned long, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @size_returning_update_test()
+define void @size_returning_update_test() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTCOLD]])
+  %call = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #3
+  %p  = extractvalue {ptr, i64} %call, 0
+  call void @dummy(ptr %p)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #4
+  %p1 = extractvalue {ptr, i64} %call1, 0
+  call void @dummy(ptr %p1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTHOT]])
+  %call2 = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #5
+  %p2 = extractvalue {ptr, i64} %call2, 0
+  call void @dummy(ptr %p2)
+  ret void
+}
+
+;; Check that __size_returning_new_aligned_hot_cold(unsigned long, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @size_returning_aligned_update_test()
+define void @size_returning_aligned_update_test() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTCOLD]])
+  %call = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #3
+  %p  = extractvalue {ptr, i64} %call, 0
+  call void @dummy(ptr %p)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #4
+  %p1 = extractvalue {ptr, i64} %call1, 0
+  call void @dummy(ptr %p1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTHOT]])
+  %call2 = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #5
+  %p2 = extractvalue {ptr, i64} %call2, 0
+  call void @dummy(ptr %p2)
+  ret void
+}
+
 ;; So that instcombine doesn't optimize out the call.
 declare void @dummy(ptr)
 
@@ -360,6 +448,18 @@ declare ptr @_ZnamSt11align_val_t12__hot_cold_t(i64, i64, i8)
 declare ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64, ptr, i8)
 declare ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64, i64, ptr, i8)
 
+
+declare {ptr, i64} @__size_returning_new(i64)
+declare {ptr, i64} @__size_returning_new_hot_cold(i64, i8)
+declare {ptr, i64} @__size_returning_new_aligned(i64, i64)
+declare {ptr, i64} @__size_returning_new_aligned_hot_cold(i64, i64, i8)
+
 attributes #0 = { builtin allocsize(0) "memprof"="cold" }
 attributes #1 = { builtin allocsize(0) "memprof"="notcold" }
 attributes #2 = { builtin allocsize(0) "memprof"="hot" }
+
+;; Use separate attributes for __size_returning_new variants since they are not
+;; treated as builtins.
+attributes #3 = { "memprof" = "cold" }
+attributes #4 = { "memprof" = "notcold" }
+attributes #5 = { "memprof" = "hot" }
diff --git a/llvm/test/Transforms/InstCombine/ucmp.ll b/llvm/test/Transforms/InstCombine/ucmp.ll
index 7210455094baacb..ad8a57825253b0f 100644
--- a/llvm/test/Transforms/InstCombine/ucmp.ll
+++ b/llvm/test/Transforms/InstCombine/ucmp.ll
@@ -157,6 +157,30 @@ define i1 @ucmp_sle_neg_1(i32 %x, i32 %y) {
   ret i1 %2
 }
 
+; ucmp(x, y) u< C => x u>= y when C u> 1 and C != -1
+define i1 @ucmp_ult_positive_const_gt_than_1_lt_than_umax(i32 %x, i32 %y) {
+; CHECK-LABEL: define i1 @ucmp_ult_positive_const_gt_than_1_lt_than_umax(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = icmp ult i8 %1, 4
+  ret i1 %2
+}
+
+; ucmp(x, y) u> C => x u< y when C != 0 and C != -1
+define i1 @ucmp_ugt_const_not_0_or_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i1 @ucmp_ugt_const_not_0_or_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = icmp ugt i8 %1, 12
+  ret i1 %2
+}
+
 ; ========== Fold -ucmp(x, y) => ucmp(y, x) ==========
 define i8 @ucmp_negated(i32 %x, i32 %y) {
 ; CHECK-LABEL: define i8 @ucmp_negated(
@@ -183,3 +207,309 @@ define i8 @ucmp_negated_multiuse(i32 %x, i32 %y) {
   %2 = sub i8 0, %1
   ret i8 %2
 }
+
+; Fold ((x u< y) ? -1 : (x != y)) into ucmp(x, y)
+define i8 @ucmp_from_select_lt(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Vector version
+define <4 x i8> @ucmp_from_select_vec_lt(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: define <4 x i8> @ucmp_from_select_vec_lt(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i8> @llvm.ucmp.v4i8.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %ne_bool = icmp ne <4 x i32> %x, %y
+  %ne = zext <4 x i1> %ne_bool to <4 x i8>
+  %lt = icmp ult <4 x i32> %x, %y
+  %r = select <4 x i1> %lt, <4 x i8> splat(i8 -1), <4 x i8> %ne
+  ret <4 x i8> %r
+}
+
+; Commuted operands
+define i8 @ucmp_from_select_lt_commuted_ops1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_commuted_ops1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %y, %x
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+define i8 @ucmp_from_select_lt_commuted_ops2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_commuted_ops2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ugt i32 %y, %x
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Negative test: false value of the select is not `icmp ne x, y`
+define i8 @ucmp_from_select_lt_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp eq i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LT:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LT]], i8 -1, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp eq i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Negative test: true value of select is not -1
+define i8 @ucmp_from_select_lt_neg2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_neg2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LT:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LT]], i8 2, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 2, i8 %ne
+  ret i8 %r
+}
+
+; Negative test: false value of select is sign-extended instead of zero-extended
+define i8 @ucmp_from_select_lt_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_neg3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Negative test: condition of select is not (x u< y)
+define i8 @ucmp_from_select_lt_neg4(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_neg4(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LT_NOT:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LT_NOT]], i8 [[NE]], i8 -1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ule i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Fold (x u<= y) ? sext(x != y) : 1 into ucmp(x, y)
+define i8 @ucmp_from_select_le(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Negative test: condition of select is not (x u<= y)
+define i8 @ucmp_from_select_le_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LE_NOT:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LE_NOT]], i8 1, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ult i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Negative test: true value of select is zero-extended instead of sign-extended
+define i8 @ucmp_from_select_le_neg2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le_neg2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Negative test: true value is not x != y
+define i8 @ucmp_from_select_le_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le_neg3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp sgt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LE_NOT:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LE_NOT]], i8 1, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp sgt i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Negative test: false value is not 1
+define i8 @ucmp_from_select_le_neg4(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le_neg4(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Fold (x u>= y) ? zext(x != y) : -1 into ucmp(x, y)
+define i8 @ucmp_from_select_ge(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Commuted operands
+define i8 @ucmp_from_select_ge_commuted_ops1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_commuted_ops1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %y, %x
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+define i8 @ucmp_from_select_ge_commuted_ops2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_commuted_ops2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp ule i32 %y, %x
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Negative test: condition is not x u>= y
+define i8 @ucmp_from_select_ge_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[GE:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[GE]], i8 [[NE]], i8 -1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp ult i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Negative test: true value is sign-extended instead of zero-extended
+define i8 @ucmp_from_select_ge_neg2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_neg2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Negative test: true value is not x != y
+define i8 @ucmp_from_select_ge_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_neg3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp sgt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[GE_NOT:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[GE_NOT]], i8 -1, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp sgt i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Negative test: false value is not -1
+define i8 @ucmp_from_select_ge_neg4(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_neg4(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[GE_NOT:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[GE_NOT]], i8 3, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 3
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/fptoi-range.ll b/llvm/test/Transforms/InstSimplify/fptoi-range.ll
index cc9605259aa811e..95f2a9d50793c64 100644
--- a/llvm/test/Transforms/InstSimplify/fptoi-range.ll
+++ b/llvm/test/Transforms/InstSimplify/fptoi-range.ll
@@ -32,6 +32,15 @@ define i1 @f16_si16_max2(half %f) {
   ret i1 %c
 }
 
+define i1 @f16_si128_max2(half %f) {
+; CHECK-LABEL: @f16_si128_max2(
+; CHECK-NEXT:    ret i1 false
+;
+  %i = fptosi half %f to i128
+  %c = icmp sgt i128 %i, 65504
+  ret i1 %c
+}
+
 define i1 @f16_si_min1(half %f) {
 ; CHECK-LABEL: @f16_si_min1(
 ; CHECK-NEXT:    ret i1 true
@@ -41,6 +50,15 @@ define i1 @f16_si_min1(half %f) {
   ret i1 %c
 }
 
+define i1 @f16_si128_min1(half %f) {
+; CHECK-LABEL: @f16_si128_min1(
+; CHECK-NEXT:    ret i1 true
+;
+  %i = fptosi half %f to i128
+  %c = icmp sge i128 %i, -65504
+  ret i1 %c
+}
+
 define i1 @f16_si16_min1(half %f) {
 ; CHECK-LABEL: @f16_si16_min1(
 ; CHECK-NEXT:    [[I:%.*]] = fptosi half [[F:%.*]] to i16
diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
index 40b22c619f76867..ea139f2411cc3df 100644
--- a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
@@ -932,3 +932,119 @@ define i1 @usub_uge_fail(i8 %x, i8 %y) {
   %cmp = icmp uge i8 %sat, %x
   ret i1 %cmp
 }
+
+define i1 @icmp_ult_uaddsat_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add(
+; CHECK-NEXT:    ret i1 false
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %add = add i32 %x, %y
+  %cmp = icmp ult i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_uge_uaddsat_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_uge_uaddsat_add(
+; CHECK-NEXT:    ret i1 true
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %add = add i32 %x, %y
+  %cmp = icmp uge i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ugt_uaddsat_add_commuted1(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ugt_uaddsat_add_commuted1(
+; CHECK-NEXT:    ret i1 false
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %add = add i32 %x, %y
+  %cmp = icmp ugt i32 %add, %uaddsat
+  ret i1 %cmp
+}
+
+define i1 @icmp_ult_uaddsat_add_commuted2(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add_commuted2(
+; CHECK-NEXT:    ret i1 false
+;
+  %xx = mul i32 %x, 998244353
+  %yy = mul i32 %y, 998244353
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %xx, i32 %yy)
+  %add = add i32 %yy, %xx ; thwart complexity-based canonicalization
+  %cmp = icmp ult i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ule_usubsat_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ule_usubsat_sub(
+; CHECK-NEXT:    ret i1 true
+;
+  %usubsat = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y)
+  %add = sub i32 %x, %y
+  %cmp = icmp ule i32 %usubsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ugt_usubsat_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ugt_usubsat_sub(
+; CHECK-NEXT:    ret i1 false
+;
+  %usubsat = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y)
+  %add = sub i32 %x, %y
+  %cmp = icmp ugt i32 %usubsat, %add
+  ret i1 %cmp
+}
+
+; Negative tests
+
+define i1 @icmp_ult_uaddsat_add_mismatch(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add_mismatch(
+; CHECK-NEXT:    [[UADDSAT:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Z:%.*]])
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[UADDSAT]], [[ADD]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %z)
+  %add = add i32 %x, %y
+  %cmp = icmp ult i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ult_uaddsat_add_wrong_pred(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add_wrong_pred(
+; CHECK-NEXT:    [[UADDSAT:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[UADDSAT]], [[ADD]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %add = add i32 %x, %y
+  %cmp = icmp ule i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ult_uaddsat_add_wrong_op(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add_wrong_op(
+; CHECK-NEXT:    [[UADDSAT:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[UADDSAT]], [[SUB]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %sub = sub i32 %x, %y
+  %cmp = icmp ult i32 %uaddsat, %sub
+  ret i1 %cmp
+}
+
+define i1 @icmp_ule_usubsat_sub_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ule_usubsat_sub_commuted(
+; CHECK-NEXT:    [[USUBSAT:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[ADD:%.*]] = sub i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[USUBSAT]], [[ADD]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %usubsat = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y)
+  %add = sub i32 %y, %x
+  %cmp = icmp ule i32 %usubsat, %add
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-cost-compare.ll b/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-cost-compare.ll
index 9c11bd064ad47c3..cadee94ff409609 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-cost-compare.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-cost-compare.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; RUN: opt < %s -passes=loop-reduce,loop-term-fold -S | FileCheck %s
 
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
 target triple = "riscv64"
diff --git a/llvm/test/Transforms/LoopStrengthReduce/RISCV/term-fold-crash.ll b/llvm/test/Transforms/LoopStrengthReduce/RISCV/term-fold-crash.ll
index 8ca7f0010bbbe5f..9fb240684d232b4 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/RISCV/term-fold-crash.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/RISCV/term-fold-crash.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S -passes=loop-reduce -mtriple=riscv64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=loop-reduce,loop-term-fold -mtriple=riscv64-unknown-linux-gnu < %s | FileCheck %s
 
 define void @test(ptr %p, i8 %arg, i32 %start) {
 ; CHECK-LABEL: define void @test(
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold-negative-testcase.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold-negative-testcase.ll
index 2d3d3a4b72a1ac9..89ddba3343ffa24 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold-negative-testcase.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold-negative-testcase.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; REQUIRES: asserts
-; RUN: opt < %s -passes="loop-reduce" -S -debug -lsr-term-fold 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-reduce,loop-term-fold -S -debug 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-n64"
 
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll
index 7299a014b798351..6f34dc843ae1ee3 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes="loop-reduce" -S -lsr-term-fold | FileCheck %s
+; RUN: opt < %s -passes="loop-reduce,loop-term-fold" -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-n64"
 
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll
index 1454535b52bccbb..67a71496e4cec8d 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -loop-reduce -S -lsr-term-fold | FileCheck %s
+; RUN: opt < %s -passes=loop-reduce,loop-term-fold -S | FileCheck %s
 
 ; This test used to crash due to matchSimpleRecurrence matching the simple
 ; recurrence in pn-loop when evaluating unrelated-loop. Since unrelated-loop
@@ -13,9 +13,10 @@ define void @phi_node_different_bb() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[TMP2]], 1
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[PN_LOOP]], label [[UNRELATED_LOOP_PREHEADER:%.*]]
 ; CHECK:       unrelated-loop.preheader:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP2]], [[PN_LOOP]] ]
 ; CHECK-NEXT:    br label [[UNRELATED_LOOP:%.*]]
 ; CHECK:       unrelated-loop:
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[DOTLCSSA]], 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[END:%.*]], label [[UNRELATED_LOOP]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
index 933b46733f9d1b6..736be4995c575fc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
@@ -49,8 +49,6 @@ define void @foo() {
 ; CHECK:       vector.latch:
 ; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x float> [ [[TMP12]], [[INNER_LOOP1]] ]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI5]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq <vscale x 4 x i64> [[TMP16]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1024, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
new file mode 100644
index 000000000000000..d970b427d035da7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -0,0 +1,850 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+target triple = "riscv64-unknown-linux-gnu"
+
+define void @block_with_dead_inst_1(ptr %src, i64 %N) #0 {
+; CHECK-LABEL: define void @block_with_dead_inst_1(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP12]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 3, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP15]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i16> zeroinitializer, i32 [[TMP24]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[DEAD_GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP25]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  %dead.gep = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %1 = icmp eq i64 %iv.next, %N
+  br i1 %1, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @block_with_dead_inst_2(ptr %src) #0 {
+; CHECK-LABEL: define void @block_with_dead_inst_2(
+; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP9]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP12]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 0, i32 [[TMP16]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP20]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> zeroinitializer, i32 [[TMP21]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[DEAD_GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 0, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %loop.latch, label %else
+
+else:
+  %dead.gep = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @multiple_blocks_with_dead_insts_3(ptr %src) #0 {
+; CHECK-LABEL: define void @multiple_blocks_with_dead_insts_3(
+; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP9]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP12]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 0, i32 [[TMP16]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP20]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> zeroinitializer, i32 [[TMP21]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[DEAD_GEP_1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[DEAD_GEP_2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 0, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %else
+
+then:
+  %dead.gep.1 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+else:
+  %dead.gep.2 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @multiple_blocks_with_dead_insts_4(ptr %src, i64 %N) #0 {
+; CHECK-LABEL: define void @multiple_blocks_with_dead_insts_4(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP12]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 3, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP15]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i16> zeroinitializer, i32 [[TMP24]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[THEN_1:.*]]
+; CHECK:       [[THEN_1]]:
+; CHECK-NEXT:    [[DEAD_GEP_1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[DEAD_GEP_2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %else
+
+then:
+  br label %then.1
+
+then.1:
+  %dead.gep.1 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+else:
+  %dead.gep.2 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @multiple_blocks_with_dead_inst_multiple_successors_5(ptr %src) #0 {
+; CHECK-LABEL: define void @multiple_blocks_with_dead_inst_multiple_successors_5(
+; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP9]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP12]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 1, i32 [[TMP16]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP20]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> zeroinitializer, i32 [[TMP21]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[THEN_1:.*]]
+; CHECK:       [[THEN_1]]:
+; CHECK-NEXT:    [[DEAD_GEP_1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[ELSE_2:.*]]
+; CHECK:       [[ELSE_2]]:
+; CHECK-NEXT:    [[DEAD_GEP_2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %else
+
+then:
+  br label %then.1
+
+then.1:
+  %dead.gep.1 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+else:
+  br label %else.2
+
+else.2:
+  %dead.gep.2 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @multiple_blocks_with_dead_inst_multiple_successors_6(ptr %src, i1 %ic, i64 %N) #0 {
+; CHECK-LABEL: define void @multiple_blocks_with_dead_inst_multiple_successors_6(
+; CHECK-SAME: ptr [[SRC:%.*]], i1 [[IC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP12]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 3, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP15]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i16> zeroinitializer, i32 [[TMP24]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br i1 [[IC]], label %[[THEN_1:.*]], label %[[ELSE]]
+; CHECK:       [[THEN_1]]:
+; CHECK-NEXT:    [[DEAD_GEP_1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[ELSE_2:.*]]
+; CHECK:       [[ELSE_2]]:
+; CHECK-NEXT:    [[DEAD_GEP_2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %else
+
+then:
+  br i1 %ic, label %then.1, label %else
+
+then.1:
+  %dead.gep.1 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+else:
+  br label %else.2
+
+else.2:
+  %dead.gep.2 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @empty_block_with_phi_1(ptr %src, i64 %N) #0 {
+; CHECK-LABEL: define void @empty_block_with_phi_1(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1, i32 [[TMP8]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i16, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP11]], align 2
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <vscale x 8 x i16> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP12]], <vscale x 8 x i16> shufflevector (<vscale x 8 x i16> insertelement (<vscale x 8 x i16> poison, i16 99, i64 0), <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer), <vscale x 8 x i16> [[WIDE_LOAD]]
+; CHECK-NEXT:    store <vscale x 8 x i16> [[PREDPHI]], ptr [[TMP11]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> zeroinitializer, i32 [[TMP16]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i32 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i16 [ [[L]], %[[LOOP_HEADER]] ], [ 99, %[[THEN]] ]
+; CHECK-NEXT:    store i16 [[P]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i32 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i32 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  br label %loop.latch
+
+loop.latch:
+  %p = phi i16 [ %l, %loop.header ], [ 99, %then ]
+  store i16 %p, ptr %gep
+  %iv.next = add nsw i64 %iv, 1
+  %1 = icmp eq i64 %iv.next, %N
+  br i1 %1, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @empty_block_with_phi_2(ptr %src, i64 %N) #0 {
+; CHECK-LABEL: define void @empty_block_with_phi_2(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1, i32 [[TMP8]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i16, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP11]], align 2
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <vscale x 8 x i16> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <vscale x 8 x i1> [[TMP12]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP13]], <vscale x 8 x i16> shufflevector (<vscale x 8 x i16> insertelement (<vscale x 8 x i16> poison, i16 99, i64 0), <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer), <vscale x 8 x i16> [[WIDE_LOAD]]
+; CHECK-NEXT:    store <vscale x 8 x i16> [[PREDPHI]], ptr [[TMP11]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 [[TMP16]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> zeroinitializer, i32 [[TMP17]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i32 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i16 [ [[L]], %[[LOOP_HEADER]] ], [ 99, %[[ELSE]] ]
+; CHECK-NEXT:    store i16 [[P]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i32 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i32 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %loop.latch, label %else
+
+else:
+  br label %loop.latch
+
+loop.latch:
+  %p = phi i16 [ %l, %loop.header ], [ 99, %else ]
+  store i16 %p, ptr %gep
+  %iv.next = add nsw i64 %iv, 1
+  %1 = icmp eq i64 %iv.next, %N
+  br i1 %1, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-features"="+64bit,+v" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
index 3f56361409b02f1..3bf2591391121ea 100644
--- a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
@@ -13,7 +13,6 @@ define void @foo(ptr %h) !dbg !4 {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]], !dbg [[DBG21]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_COND_CLEANUP32:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_COND_CLEANUP32]] ]
 ; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER1:%.*]], !dbg [[DBG21]]
 ; CHECK:       for.cond5.preheader1:
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP4:%.*]], [[FOR_COND5_PREHEADER1]] ], !dbg [[DBG21]]
@@ -30,10 +29,7 @@ define void @foo(ptr %h) !dbg !4 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0, !dbg [[DBG26:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP32]], label [[FOR_COND5_PREHEADER1]], !dbg [[DBG26]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[TMP7:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1>, !dbg [[DBG27:![0-9]+]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i64> [[TMP7]], <i64 23, i64 23, i64 23, i64 23>, !dbg [[DBG28:![0-9]+]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK:       middle.block:
@@ -59,9 +55,9 @@ define void @foo(ptr %h) !dbg !4 {
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC10]], 5, !dbg [[DBG25]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_COND5_PREHEADER]], !dbg [[DBG26]]
 ; CHECK:       for.cond.cleanup3:
-; CHECK-NEXT:    [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG27]]
+; CHECK-NEXT:    [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG27:![0-9]+]]
 ; CHECK-NEXT:      #dbg_value(i64 [[INC13]], [[META11]], !DIExpression(), [[META20]])
-; CHECK-NEXT:    [[EXITCOND24_NOT:%.*]] = icmp eq i64 [[INC13]], 23, !dbg [[DBG28]]
+; CHECK-NEXT:    [[EXITCOND24_NOT:%.*]] = icmp eq i64 [[INC13]], 23, !dbg [[DBG28:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[EXITCOND24_NOT]], label [[EXIT]], label [[FOR_COND1_PREHEADER]], !dbg [[DBG21]], !llvm.loop [[LOOP34:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void, !dbg [[DBG35:![0-9]+]]
@@ -164,13 +160,13 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 ; CHECK: [[DBG24]] = !DILocation(line: 11, column: 32, scope: [[META19]])
 ; CHECK: [[DBG25]] = !DILocation(line: 11, column: 26, scope: [[META19]])
 ; CHECK: [[DBG26]] = !DILocation(line: 11, column: 5, scope: [[META15]])
-; CHECK: [[DBG27]] = !DILocation(line: 10, column: 30, scope: [[META16]])
-; CHECK: [[DBG28]] = !DILocation(line: 10, column: 24, scope: [[META16]])
 ; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[DBG21]], [[META30:![0-9]+]], [[META31:![0-9]+]], [[META32:![0-9]+]]}
 ; CHECK: [[META30]] = !DILocation(line: 13, column: 13, scope: [[META12]])
 ; CHECK: [[META31]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META32]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[DBG33]] = !DILocation(line: 13, column: 2, scope: [[META23]])
+; CHECK: [[DBG27]] = !DILocation(line: 10, column: 30, scope: [[META16]])
+; CHECK: [[DBG28]] = !DILocation(line: 10, column: 24, scope: [[META16]])
 ; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[DBG21]], [[META30]], [[META31]]}
 ; CHECK: [[DBG35]] = !DILocation(line: 14, column: 1, scope: [[DBG4]])
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
index 260f59c5c97e274..b6f2ae3c14893a5 100644
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
@@ -41,8 +41,6 @@ define void @test(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[LOOP_1_LATCH5]], label [[LOOP_2_HEADER1]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[TMP8]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
index 986a6082a6891db..5df4fce2190a3a6 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
@@ -52,8 +52,6 @@ define void @non_outermost_loop_hcfg_construction(i64 %n, ptr %a) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_LOOP_LATCH4]], label [[INNERMOST_LOOP1]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[TMP8]], [[BROADCAST_SPLAT3]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -197,8 +195,6 @@ define void @non_outermost_loop_hcfg_construction_other_loops_at_same_level(i64
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_LOOP_J0_CLEANUP4]], label [[INNERMOST_LOOP1]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[TMP8]], [[BROADCAST_SPLAT3]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
index 6cd7e3714201b7a..08f2b823815b1ce 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
@@ -55,8 +55,6 @@ define void @foo() {
 ; CHECK:       vector.latch:
 ; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x float> [ [[TMP12]], [[INNER_LOOP1]] ]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI5]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq <vscale x 4 x i64> [[TMP16]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1024, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll b/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll
index 0c9fa742765d311..8104441ad19934b 100644
--- a/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll
+++ b/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll
@@ -14,6 +14,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 ; CHECK: LV: User VF=4 is unsafe, clamping to max safe VF=2.
 ; CHECK: remark: <unknown>:0:0: User-specified vectorization factor 4 is unsafe, clamping to maximum safe vectorization factor 2
+; CHECK: remark: <unknown>:0:0: UserVF ignored because it may be larger than the maximal safe VF
 ; CHECK-LABEL: @foo
 ; CHECK: <2 x i32>
 define void @foo(ptr %a, ptr %b) {
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll b/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
index fdf73963e86464a..1552af3140e3837 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
@@ -40,8 +40,6 @@ define void @inner_loop_reduction(ptr noalias nocapture readonly %a.in, ptr noal
 ; CHECK-NEXT: %[[REDUCTION:.*]] = phi <4 x double> [ %[[REDUCTION_NEXT]], %[[FOR2_HEADER]] ]
 ; CHECK-NEXT: %[[C_PTR:.*]] = getelementptr inbounds double, ptr %c.out, <4 x i64> %[[VEC_INDEX]]
 ; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> %[[REDUCTION]], <4 x ptr> %[[C_PTR]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; CHECK-NEXT: %[[VEC_INDEX_NEXT:.*]] = add nuw nsw <4 x i64> %[[VEC_INDEX]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT: %[[VEC_PTR:.*]] = icmp eq <4 x i64> %[[VEC_INDEX_NEXT]], <i64 1000, i64 1000, i64 1000, i64 1000>
 ; CHECK-NEXT: %[[FOR1_INDEX_NEXT:.*]] = add nuw i64 %[[FOR1_INDEX]], 4
 ; CHECK-NEXT: %{{.*}} = add <4 x i64> %[[VEC_INDEX]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT: %[[EXIT_COND:.*]] = icmp eq i64 %[[FOR1_INDEX_NEXT]], 1000
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll b/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
index 3335c21c8d74571..4241409d3935fd6 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
@@ -30,8 +30,6 @@ define void @widen_call_instruction(ptr noalias nocapture readonly %a.in, ptr no
 ; CHECK-NEXT: %[[REDUCTION:.*]] = phi <4 x double> [ %[[REDUCTION_NEXT]], %[[FOR2_HEADER]] ]
 ; CHECK-NEXT: %[[C_PTR:.*]] = getelementptr inbounds double, ptr %c.out, <4 x i64> %[[VEC_INDEX]]
 ; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> %[[REDUCTION]], <4 x ptr> %[[C_PTR]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; CHECK-NEXT: %[[VEC_INDEX_NEXT:.*]] = add nuw nsw <4 x i64> %[[VEC_INDEX]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT: %[[VEC_PTR:.*]] = icmp eq <4 x i64> %[[VEC_INDEX_NEXT]], <i64 1000, i64 1000, i64 1000, i64 1000>
 ; CHECK-NEXT: %[[FOR1_INDEX_NEXT:.*]] = add nuw i64 %[[FOR1_INDEX]], 4
 ; CHECK-NEXT: %{{.*}} = add <4 x i64> %[[VEC_INDEX]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT: %[[EXIT_COND:.*]] = icmp eq i64 %[[FOR1_INDEX_NEXT]], 1000
diff --git a/llvm/test/Transforms/OpenMP/barrier_removal.ll b/llvm/test/Transforms/OpenMP/barrier_removal.ll
index 0925418cf54c725..47a5d5104aa8bd3 100644
--- a/llvm/test/Transforms/OpenMP/barrier_removal.ll
+++ b/llvm/test/Transforms/OpenMP/barrier_removal.ll
@@ -269,10 +269,9 @@ define void @neg_empty_2() "kernel" {
 define void @pos_constant_loads() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_constant_loads
 ; CHECK-SAME: () #[[ATTR4]] {
-; CHECK-NEXT:    [[ARG:%.*]] = load ptr addrspace(4), ptr addrspacecast (ptr addrspace(4) @GPtr4 to ptr), align 8
-; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspacecast (ptr addrspace(4) @GC2 to ptr), align 4
-; CHECK-NEXT:    [[ARGC:%.*]] = addrspacecast ptr addrspace(4) [[ARG]] to ptr
-; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[ARGC]], align 4
+; CHECK-NEXT:    [[ARG:%.*]] = load ptr addrspace(4), ptr addrspace(4) @GPtr4, align 8
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspace(4) @GC2, align 4
+; CHECK-NEXT:    [[C:%.*]] = load i32, ptr addrspace(4) [[ARG]], align 4
 ; CHECK-NEXT:    [[D:%.*]] = add i32 42, [[B]]
 ; CHECK-NEXT:    [[E:%.*]] = add i32 [[D]], [[C]]
 ; CHECK-NEXT:    call void @useI32(i32 [[E]])
@@ -303,7 +302,7 @@ define void @neg_loads() "kernel" {
 ; CHECK-NEXT:    [[ARG:%.*]] = load ptr, ptr @GPtr, align 8
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr @G, align 4
 ; CHECK-NEXT:    call void @aligned_barrier()
-; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspacecast (ptr addrspace(3) @GS to ptr), align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspace(3) @GS, align 4
 ; CHECK-NEXT:    call void @aligned_barrier()
 ; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[ARG]], align 4
 ; CHECK-NEXT:    call void @aligned_barrier()
@@ -335,9 +334,8 @@ define void @pos_priv_mem() "kernel" {
 ; CHECK-NEXT:    [[LOC:%.*]] = alloca i32, align 4, addrspace(5)
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr @PG1, align 4
 ; CHECK-NEXT:    store i32 [[A]], ptr addrspace(5) [[LOC]], align 4
-; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspacecast (ptr addrspace(5) @PG2 to ptr), align 4
-; CHECK-NEXT:    [[ARGC:%.*]] = addrspacecast ptr addrspace(5) [[ARG]] to ptr
-; CHECK-NEXT:    store i32 [[B]], ptr [[ARGC]], align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspace(5) @PG2, align 4
+; CHECK-NEXT:    store i32 [[B]], ptr addrspace(5) [[ARG]], align 4
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr addrspace(5) [[LOC]], align 4
 ; CHECK-NEXT:    store i32 [[V]], ptr @PG1, align 4
 ; CHECK-NEXT:    ret void
@@ -370,7 +368,7 @@ define void @neg_mem() "kernel" {
 ; CHECK-NEXT:    store i32 [[A]], ptr [[ARG]], align 4
 ; CHECK-NEXT:    fence release
 ; CHECK-NEXT:    call void @aligned_barrier()
-; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspacecast (ptr addrspace(1) @G2 to ptr), align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspace(1) @G2, align 4
 ; CHECK-NEXT:    store i32 [[B]], ptr @G1, align 4
 ; CHECK-NEXT:    fence acquire
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/OpenMP/heap-to-shared-missing-declarations.ll b/llvm/test/Transforms/OpenMP/heap-to-shared-missing-declarations.ll
index 7a5bba75f6c48c5..d81f34f3c427374 100644
--- a/llvm/test/Transforms/OpenMP/heap-to-shared-missing-declarations.ll
+++ b/llvm/test/Transforms/OpenMP/heap-to-shared-missing-declarations.ll
@@ -22,7 +22,7 @@ define internal void @func() {
 ; CHECK-LABEL: define {{[^@]+}}@func
 ; CHECK-SAME: () #[[ATTR1]] {
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[I:%.*]] = load ptr, ptr null, align 4294967296
+; CHECK-NEXT:    [[I:%.*]] = load ptr, ptr addrspace(5) null, align 4294967296
 ; CHECK-NEXT:    store i64 0, ptr [[I]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/OpenMP/nested_parallelism.ll b/llvm/test/Transforms/OpenMP/nested_parallelism.ll
index 4f4a87cbddfec15..5c4386b24a3d5a6 100644
--- a/llvm/test/Transforms/OpenMP/nested_parallelism.ll
+++ b/llvm/test/Transforms/OpenMP/nested_parallelism.ll
@@ -60,7 +60,7 @@ define weak_odr protected void @__omp_offloading_10302_bd7e0_main_l13(ptr %dyn,
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[REGION_GUARDED_I:%.*]], label [[_Z3FOOI_INTERNALIZED_EXIT:%.*]]
 ; CHECK:       region.guarded.i:
 ; CHECK-NEXT:    [[I_ADDR_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[I:%.*]] to i32
-; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspacecast (ptr addrspace(3) @i_shared to ptr), align 16
+; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(3) @i_shared, align 16
 ; CHECK-NEXT:    br label [[_Z3FOOI_INTERNALIZED_EXIT]]
 ; CHECK:       _Z3fooi.internalized.exit:
 ; CHECK-NEXT:    tail call void @__kmpc_barrier_simple_spmd(ptr nonnull @[[GLOB1]], i32 [[TMP2]]) #[[ATTR2]]
@@ -140,7 +140,7 @@ define weak_odr protected void @__omp_offloading_10302_bd7e0_main_l16(ptr %dyn,
 ; CHECK-NEXT:    [[I_ADDR_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[I:%.*]] to i32
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
-; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), align 16
+; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(3) @i.i_shared, align 16
 ; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), ptr [[CAPTURED_VARS_ADDRS_I]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I]], i64 1)
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
diff --git a/llvm/test/Transforms/OpenMP/spmdization_kernel_env_dep.ll b/llvm/test/Transforms/OpenMP/spmdization_kernel_env_dep.ll
index ce7b4f89b893ffc..52be16c41f872d9 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_kernel_env_dep.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_kernel_env_dep.ll
@@ -27,21 +27,22 @@ define i32 @fputs() {
 define internal i32 @__kmpc_target_init(ptr %0, ptr %dyn) {
 ; AMDGPU-LABEL: define {{[^@]+}}@__kmpc_target_init
 ; AMDGPU-SAME: (ptr [[TMP0:%.*]], ptr [[DYN:%.*]]) #[[ATTR1:[0-9]+]] {
-; AMDGPU-NEXT:    [[TMP2:%.*]] = load i8, ptr getelementptr (i8, ptr addrspacecast (ptr addrspace(1) @__omp_offloading_10302_b20a40e_main_l4_kernel_environment to ptr), i64 2), align 2
-; AMDGPU-NEXT:    [[TMP3:%.*]] = and i8 [[TMP2]], 2
-; AMDGPU-NEXT:    [[TMP4:%.*]] = icmp ne i8 [[TMP3]], 0
-; AMDGPU-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x() #[[ATTR3:[0-9]+]]
-; AMDGPU-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; AMDGPU-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP4]], i1 [[TMP6]], i1 false
-; AMDGPU-NEXT:    br i1 [[OR_COND]], label [[TMP7:%.*]], label [[TMP8:%.*]]
-; AMDGPU:       7:
-; AMDGPU-NEXT:    store i8 0, ptr addrspace(3) null, align 2147483648
-; AMDGPU-NEXT:    br label [[TMP8]]
+; AMDGPU-NEXT:    [[TMP2:%.*]] = addrspacecast ptr getelementptr (i8, ptr addrspacecast (ptr addrspace(1) @__omp_offloading_10302_b20a40e_main_l4_kernel_environment to ptr), i64 2) to ptr addrspace(1)
+; AMDGPU-NEXT:    [[TMP3:%.*]] = load i8, ptr addrspace(1) [[TMP2]], align 2
+; AMDGPU-NEXT:    [[TMP4:%.*]] = and i8 [[TMP3]], 2
+; AMDGPU-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; AMDGPU-NEXT:    [[TMP6:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x() #[[ATTR3:[0-9]+]]
+; AMDGPU-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; AMDGPU-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP5]], i1 [[TMP7]], i1 false
+; AMDGPU-NEXT:    br i1 [[OR_COND]], label [[TMP8:%.*]], label [[TMP9:%.*]]
 ; AMDGPU:       8:
-; AMDGPU-NEXT:    br label [[TMP10:%.*]]
+; AMDGPU-NEXT:    store i8 0, ptr addrspace(3) null, align 2147483648
+; AMDGPU-NEXT:    br label [[TMP9]]
 ; AMDGPU:       9:
-; AMDGPU-NEXT:    unreachable
+; AMDGPU-NEXT:    br label [[TMP11:%.*]]
 ; AMDGPU:       10:
+; AMDGPU-NEXT:    unreachable
+; AMDGPU:       11:
 ; AMDGPU-NEXT:    ret i32 0
 ;
   %2 = getelementptr %struct.ConfigurationEnvironmentTy.8, ptr %0, i64 0, i32 2
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
index 99c7762a67dfbd6..454780153b82368 100644
--- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
@@ -1,4 +1,4 @@
-; RUN: not opt -passes=pgo-instr-gen,ctx-instr-lower -profile-context-root=good \
+; RUN: not opt -passes=ctx-instr-gen,ctx-instr-lower -profile-context-root=good \
 ; RUN:   -profile-context-root=bad \
 ; RUN:   -S < %s 2>&1 | FileCheck %s
 
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
index a70f94e1521f0de..df4e467567c46ef 100644
--- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 4
-; RUN: opt -passes=pgo-instr-gen -profile-context-root=an_entrypoint \
+; RUN: opt -passes=ctx-instr-gen -profile-context-root=an_entrypoint \
 ; RUN:   -S < %s | FileCheck --check-prefix=INSTRUMENT %s
-; RUN: opt -passes=pgo-instr-gen,assign-guid,ctx-instr-lower -profile-context-root=an_entrypoint \
+; RUN: opt -passes=ctx-instr-gen,assign-guid,ctx-instr-lower -profile-context-root=an_entrypoint \
 ; RUN:   -profile-context-root=another_entrypoint_no_callees \
 ; RUN:   -S < %s | FileCheck --check-prefix=LOWERING %s
 
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
index c6126727598ef28..8b82f21e38c931b 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
@@ -15,7 +15,7 @@ define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockS
 ; CHECK-NEXT:    br i1 [[CMP_NOT2]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
 ; CHECK:       while.body.preheader:
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[BLOCKSIZE]], 8
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[WHILE_BODY_PREHEADER16:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[WHILE_BODY_PREHEADER18:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[BLOCKSIZE]], -8
 ; CHECK-NEXT:    [[IND_END:%.*]] = and i32 [[BLOCKSIZE]], 7
@@ -28,27 +28,27 @@ define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockS
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRCA]], i32 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[TMP4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[PSRCB]], i32 [[TMP5]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRCA]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[OFFSET_IDX13:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX13]]
+; CHECK-NEXT:    [[OFFSET_IDX15:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[PSRCB]], i32 [[OFFSET_IDX15]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
-; CHECK-NEXT:    [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[NEXT_GEP14]], align 2
-; CHECK-NEXT:    [[TMP7:%.*]] = sext <8 x i16> [[WIDE_LOAD15]] to <8 x i32>
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nsw <8 x i32> [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    [[TMP9:%.*]] = ashr <8 x i32> [[TMP8]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
-; CHECK-NEXT:    [[TMP10:%.*]] = tail call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP9]], <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>)
-; CHECK-NEXT:    [[TMP11:%.*]] = trunc <8 x i32> [[TMP10]] to <8 x i16>
-; CHECK-NEXT:    store <8 x i16> [[TMP11]], ptr [[NEXT_GEP13]], align 2
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; CHECK-NEXT:    [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[NEXT_GEP16]], align 2
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nsw <8 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = ashr <8 x i32> [[TMP5]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP6]], <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>)
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <8 x i32> [[TMP7]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[TMP8]], ptr [[NEXT_GEP14]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[BLOCKSIZE]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[WHILE_END]], label [[WHILE_BODY_PREHEADER16]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[WHILE_END]], label [[WHILE_BODY_PREHEADER18]]
 ; CHECK:       while.body.preheader18:
 ; CHECK-NEXT:    [[BLKCNT_06_PH:%.*]] = phi i32 [ [[BLOCKSIZE]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    [[PSRCA_ADDR_05_PH:%.*]] = phi ptr [ [[PSRCA]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END7]], [[MIDDLE_BLOCK]] ]
@@ -56,16 +56,16 @@ define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockS
 ; CHECK-NEXT:    [[PSRCB_ADDR_03_PH:%.*]] = phi ptr [ [[PSRCB]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[BLKCNT_06:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BLKCNT_06_PH]], [[WHILE_BODY_PREHEADER16]] ]
-; CHECK-NEXT:    [[PSRCA_ADDR_05:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRCA_ADDR_05_PH]], [[WHILE_BODY_PREHEADER16]] ]
-; CHECK-NEXT:    [[PDST_ADDR_04:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], [[WHILE_BODY]] ], [ [[PDST_ADDR_04_PH]], [[WHILE_BODY_PREHEADER16]] ]
-; CHECK-NEXT:    [[PSRCB_ADDR_03:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[WHILE_BODY]] ], [ [[PSRCB_ADDR_03_PH]], [[WHILE_BODY_PREHEADER16]] ]
+; CHECK-NEXT:    [[BLKCNT_06:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BLKCNT_06_PH]], [[WHILE_BODY_PREHEADER18]] ]
+; CHECK-NEXT:    [[PSRCA_ADDR_05:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRCA_ADDR_05_PH]], [[WHILE_BODY_PREHEADER18]] ]
+; CHECK-NEXT:    [[PDST_ADDR_04:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], [[WHILE_BODY]] ], [ [[PDST_ADDR_04_PH]], [[WHILE_BODY_PREHEADER18]] ]
+; CHECK-NEXT:    [[PSRCB_ADDR_03:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[WHILE_BODY]] ], [ [[PSRCB_ADDR_03_PH]], [[WHILE_BODY_PREHEADER18]] ]
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRCA_ADDR_05]], i32 2
-; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[PSRCA_ADDR_05]], align 2
-; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = load i16, ptr [[PSRCA_ADDR_05]], align 2
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP10]] to i32
 ; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[PSRCB_ADDR_03]], i32 2
-; CHECK-NEXT:    [[TMP14:%.*]] = load i16, ptr [[PSRCB_ADDR_03]], align 2
-; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP14]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[PSRCB_ADDR_03]], align 2
+; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP11]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
 ; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[MUL]], 15
 ; CHECK-NEXT:    [[SPEC_SELECT_I:%.*]] = tail call i32 @llvm.smin.i32(i32 [[SHR]], i32 32767)
diff --git a/llvm/test/Transforms/PhaseOrdering/partialord-ule.ll b/llvm/test/Transforms/PhaseOrdering/partialord-ule.ll
index 23a82c29012d967..6d4a8e31edbec05 100644
--- a/llvm/test/Transforms/PhaseOrdering/partialord-ule.ll
+++ b/llvm/test/Transforms/PhaseOrdering/partialord-ule.ll
@@ -6,8 +6,8 @@
 define i1 @ule(i32 %a, i32 %b) {
 ; CHECK-LABEL: @ule(
 ; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp ule i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    ret i1 [[DOTNOT]]
+; CHECK-NEXT:    [[SWITCH_SELECTCMP1:%.*]] = icmp ule i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[SWITCH_SELECTCMP1]]
 ;
 start:
   %cmp1 = icmp eq i32 %a, %b
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/full-non-schedulable-overlap.ll b/llvm/test/Transforms/SLPVectorizer/X86/full-non-schedulable-overlap.ll
new file mode 100644
index 000000000000000..231a9512c9ee99d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/full-non-schedulable-overlap.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(double %v) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: double [[V:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> <double 0.000000e+00, double poison>, double [[V]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x double> zeroinitializer, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[T50_02:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP8:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x double> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP10:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd double [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8]] = fadd double [[TMP7]], [[V]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul <2 x double> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    [[TMP10]] = fadd <2 x double> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    br label %[[LOOP]]
+;
+entry:
+  %mul.3 = fmul double 0.000000e+00, %v
+  %mul.4 = fmul double 0.000000e+00, 0.000000e+00
+  br label %loop
+
+loop:
+  %t48.0 = phi double [ 0.000000e+00, %entry ], [ %5, %loop ]
+  %t50.02 = phi double [ 0.000000e+00, %entry ], [ %3, %loop ]
+  %t52.0 = phi double [ 0.000000e+00, %entry ], [ %7, %loop ]
+  %0 = fmul double %t52.0, %mul.3
+  %1 = fmul double %t48.0, %mul.4
+  %2 = fadd double %1, %0
+  %3 = fadd double %2, %v
+  %4 = fmul double 0.000000e+00, %mul.3
+  %5 = fadd double 0.000000e+00, %4
+  %6 = fmul double 0.000000e+00, %mul.4
+  %7 = fadd double 0.000000e+00, %6
+  br label %loop
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll b/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
new file mode 100644
index 000000000000000..5fcac3fbf3bafea
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s -slp-threshold=-10 | FileCheck %s
+
+define i64 @src(i32 %a) {
+; CHECK-LABEL: define i64 @src(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[A]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i64> [[TMP3]], <i64 4294967297, i64 4294967297, i64 4294967297, i64 4294967297>
+; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i64> [[TMP4]], <i64 1, i64 1, i64 1, i64 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP6]])
+; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP16]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP18]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> <i64 poison, i64 4294967297>, i64 [[TMP17]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = add <2 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    ret i64 [[TMP21]]
+;
+entry:
+  %0 = sext i32 %a to i64
+  %1 = add nsw i64 %0, 4294967297
+  %2 = sext i32 %a to i64
+  %3 = add nsw i64 %2, 4294967297
+  %4 = add i64 %3, %1
+  %5 = and i64 %3, 1
+  %6 = add i64 %4, %5
+  %7 = sext i32 %a to i64
+  %8 = add nsw i64 %7, 4294967297
+  %9 = add i64 %8, %6
+  %10 = and i64 %8, 1
+  %11 = add i64 %9, %10
+  %12 = sext i32 %a to i64
+  %13 = add nsw i64 %12, 4294967297
+  %14 = add i64 %13, %11
+  %15 = and i64 %13, 1
+  %16 = add i64 %14, %15
+  %17 = sext i32 %a to i64
+  %18 = add nsw i64 %17, 4294967297
+  %19 = add i64 %18, %16
+  %20 = and i64 %18, 1
+  %21 = add i64 %19, %20
+  ret i64 %21
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 59201da1d9ac1a4..ad1a57c71761975 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -336,3 +336,31 @@ entry:
   store <4 x i32> %vmovl.i108, ptr %add.ptr35, align 4
   ret void
 }
+
+define void @test11(<2 x i64> %0, i64 %1, <2 x i64> %2) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i64> [[TMP0:%.*]], i64 [[TMP1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> <i64 5, i64 0>, [[TMP2:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i16> @llvm.vector.insert.v4i16.v2i16(<4 x i16> poison, <2 x i16> [[TMP5]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i16> @llvm.vector.insert.v4i16.v2i16(<4 x i16> [[TMP6]], <2 x i16> [[TMP7]], i64 2)
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i16> [[TMP8]] to <4 x i8>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i8> @llvm.vector.insert.v4i8.v2i8(<4 x i8> poison, <2 x i8> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i8> @llvm.vector.insert.v4i8.v2i8(<4 x i8> [[TMP10]], <2 x i8> zeroinitializer, i64 2)
+; CHECK-NEXT:    [[TMP12:%.*]] = urem <4 x i8> [[TMP9]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <4 x i8> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %3 = insertelement <2 x i64> %0, i64 %1, i32 1
+  %4 = add <2 x i64> <i64 5, i64 0>, %2
+  %5 = trunc <2 x i64> %3 to <2 x i8>
+  %6 = trunc <2 x i64> %4 to <2 x i8>
+  %7 = urem <2 x i8> %5, zeroinitializer
+  %8 = urem <2 x i8> %6, zeroinitializer
+  %9 = icmp ne <2 x i8> %7, zeroinitializer
+  %10 = icmp ne <2 x i8> %8, zeroinitializer
+  ret void
+}
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expamd-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expamd-masked-load.ll
new file mode 100644
index 000000000000000..35e5bcde4c0dbd2
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expamd-masked-load.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,
+; COM: since, on GPUs, each i1 takes up at least one register and so they should
+; COM: be treated separately.
+
+define <2 x i32> @scalarize_v2i32(ptr %p, <2 x i1> %mask, <2 x i32> %passthru) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[PASSTHRU:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_LOAD:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[PASSTHRU]], i32 [[TMP4]], i64 0
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i32> [ [[TMP5]], %[[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[COND_LOAD1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_LOAD1]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> [[RES_PHI_ELSE]], i32 [[TMP9]], i64 1
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i32> [ [[TMP10]], %[[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, i32 128, <2 x i1> %mask, <2 x i32> %passthru)
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @scalarize_v2i32_splat_mask(ptr %p, i1 %mask, <2 x i32> %passthrough) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32_splat_mask(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[MASK:%.*]], <2 x i32> [[PASSTHROUGH:%.*]]) {
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_LOAD:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[RET_COND_LOAD:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[RET:%.*]] = phi <2 x i32> [ [[RET_COND_LOAD]], %[[COND_LOAD]] ], [ [[PASSTHROUGH]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, i32 8, <2 x i1> %mask.splat, <2 x i32> %passthrough)
+  ret <2 x i32> %ret
+}
+
+define <2 x half> @scalarize_v2f16(ptr %p, <2 x i1> %mask, <2 x half> %passthru) {
+; CHECK-LABEL: define <2 x half> @scalarize_v2f16(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x half> [[PASSTHRU:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_LOAD:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds half, ptr [[P]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load half, ptr [[TMP3]], align 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x half> [[PASSTHRU]], half [[TMP4]], i64 0
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x half> [ [[TMP5]], %[[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[COND_LOAD1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_LOAD1]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds half, ptr [[P]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load half, ptr [[TMP8]], align 2
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x half> [[RES_PHI_ELSE]], half [[TMP9]], i64 1
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x half> [ [[TMP10]], %[[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x half> @llvm.masked.load.v2f16.p0(ptr %p, i32 128, <2 x i1> %mask, <2 x half> %passthru)
+  ret <2 x half> %ret
+}
+
+define <2 x i32> @scalarize_v2i32_p3(ptr addrspace(3) %p, <2 x i1> %mask, <2 x i32> %passthru) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32_p3(
+; CHECK-SAME: ptr addrspace(3) [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[PASSTHRU:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_LOAD:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(3) [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[PASSTHRU]], i32 [[TMP4]], i64 0
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i32> [ [[TMP5]], %[[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[COND_LOAD1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_LOAD1]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr addrspace(3) [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> [[RES_PHI_ELSE]], i32 [[TMP9]], i64 1
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i32> [ [[TMP10]], %[[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p3(ptr addrspace(3) %p, i32 128, <2 x i1> %mask, <2 x i32> %passthru)
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @scalarize_v2i32_lane_mask(ptr %p, <2 x i32> %passthrough) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32_lane_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[PASSTHROUGH:%.*]]) {
+; CHECK-NEXT:    [[ITEM_ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[MASK:%.*]] = icmp ult i32 [[ITEM_ID]], 16
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_LOAD:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[RET_COND_LOAD:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[RET:%.*]] = phi <2 x i32> [ [[RET_COND_LOAD]], %[[COND_LOAD]] ], [ [[PASSTHROUGH]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %item.id = call i32 @llvm.amdgcn.workitem.id.x()
+  %mask = icmp ult i32 %item.id, 16
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, i32 8, <2 x i1> %mask.splat, <2 x i32> %passthrough)
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @scalarize_v2i32_group_mask(ptr %p, <2 x i32> %passthrough) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32_group_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[PASSTHROUGH:%.*]]) {
+; CHECK-NEXT:    [[GROUP_ID:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+; CHECK-NEXT:    [[MASK:%.*]] = icmp ult i32 [[GROUP_ID]], 4
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_LOAD:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[RET_COND_LOAD:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[RET:%.*]] = phi <2 x i32> [ [[RET_COND_LOAD]], %[[COND_LOAD]] ], [ [[PASSTHROUGH]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %group.id = call i32 @llvm.amdgcn.workgroup.id.x()
+  %mask = icmp ult i32 %group.id, 4
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, i32 8, <2 x i1> %mask.splat, <2 x i32> %passthrough)
+  ret <2 x i32> %ret
+}
+
+declare <2 x i32> @llvm.masked.load.v2i32.p0(ptr, i32, <2 x i1>, <2 x i32>)
+declare <2 x half> @llvm.masked.load.v2f16.p0(ptr, i32, <2 x i1>, <2 x half>)
+declare <2 x i32> @llvm.masked.load.v2i32.p3(ptr addrspace(3), i32, <2 x i1>, <2 x i32>)
+declare noundef i32 @llvm.amdgcn.workitem.id.x()
+declare noundef i32 @llvm.amdgcn.workgroup.id.x()
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-gather.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-gather.ll
new file mode 100644
index 000000000000000..94d0e2943d93664
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-gather.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,
+; COM: since, on GPUs, each i1 takes up at least one register and so they should
+; COM: be treated separately.
+
+define <2 x i32> @scalarize_v2i32(<2 x ptr> %p, <2 x i1> %mask, <2 x i32> %passthru) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32(
+; CHECK-SAME: <2 x ptr> [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[PASSTHRU:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_LOAD:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x ptr> [[P]], i64 0
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i32, ptr [[PTR0]], align 8
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <2 x i32> [[PASSTHRU]], i32 [[LOAD0]], i64 0
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i32> [ [[RES0]], %[[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[COND_LOAD1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_LOAD1]]:
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, ptr [[PTR1]], align 8
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <2 x i32> [[RES_PHI_ELSE]], i32 [[LOAD1]], i64 1
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i32> [ [[RES1]], %[[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> %p, i32 8, <2 x i1> %mask, <2 x i32> %passthru)
+  ret <2 x i32> %ret
+}
+
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i32>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-scatter.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-scatter.ll
new file mode 100644
index 000000000000000..45debf35d06e4f3
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-scatter.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,
+; COM: since, on GPUs, each i1 takes up at least one register and so they should
+; COM: be treated separately.
+
+define void @scalarize_v2i32(<2 x ptr> %p, <2 x i1> %mask, <2 x i32> %value) {
+; CHECK-LABEL: define void @scalarize_v2i32(
+; CHECK-SAME: <2 x ptr> [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[VALUE:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <2 x i32> [[VALUE]], i64 0
+; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x ptr> [[P]], i64 0
+; CHECK-NEXT:    store i32 [[ELT0]], ptr [[PTR0]], align 8
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_STORE1]]:
+; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x i32> [[VALUE]], i64 1
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
+; CHECK-NEXT:    store i32 [[ELT1]], ptr [[PTR1]], align 8
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> %value, <2 x ptr> %p, i32 8, <2 x i1> %mask)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2i32.v2p0(<2 x i32>, <2 x ptr>, i32, <2 x i1>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-store.ll
new file mode 100644
index 000000000000000..1efd008b77e1c06
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-store.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,
+; COM: since, on GPUs, each i1 takes up at least one register and so they should
+; COM: be treated separately.
+
+define void @scalarize_v2i32(ptr %p, <2 x i1> %mask, <2 x i32> %data) {
+; CHECK-LABEL: define void @scalarize_v2i32(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 0
+; CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP4]], align 4
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_STORE1]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
+; CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP8]], align 4
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 128, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i32_splat_mask(ptr %p, <2 x i32> %data, i1 %mask) {
+; CHECK-LABEL: define void @scalarize_v2i32_splat_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]], i1 [[MASK:%.*]]) {
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    store <2 x i32> [[DATA]], ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret void
+;
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)
+  ret void
+}
+
+define void @scalarize_v2f16(ptr %p, <2 x i1> %mask, <2 x half> %data) {
+; CHECK-LABEL: define void @scalarize_v2f16(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x half> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x half> [[DATA]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds half, ptr [[P]], i32 0
+; CHECK-NEXT:    store half [[TMP3]], ptr [[TMP4]], align 2
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_STORE1]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x half> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds half, ptr [[P]], i32 1
+; CHECK-NEXT:    store half [[TMP7]], ptr [[TMP8]], align 2
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2f16.p0(<2 x half> %data, ptr %p, i32 128, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i32_p3(ptr addrspace(3) %p, <2 x i1> %mask, <2 x i32> %data) {
+; CHECK-LABEL: define void @scalarize_v2i32_p3(
+; CHECK-SAME: ptr addrspace(3) [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 0
+; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(3) [[TMP4]], align 4
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_STORE1]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 1
+; CHECK-NEXT:    store i32 [[TMP7]], ptr addrspace(3) [[TMP8]], align 4
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i32.p3(<2 x i32> %data, ptr addrspace(3) %p, i32 128, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i32_lane_mask(ptr %p, <2 x i32> %data) {
+; CHECK-LABEL: define void @scalarize_v2i32_lane_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[ITEM_ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[MASK:%.*]] = icmp ult i32 [[ITEM_ID]], 16
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    store <2 x i32> [[DATA]], ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret void
+;
+  %item.id = call i32 @llvm.amdgcn.workitem.id.x()
+  %mask = icmp ult i32 %item.id, 16
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)
+  ret void
+}
+
+define void @scalarize_v2i32_group_mask(ptr %p, <2 x i32> %data) {
+; CHECK-LABEL: define void @scalarize_v2i32_group_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[ITEM_ID:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+; CHECK-NEXT:    [[MASK:%.*]] = icmp ult i32 [[ITEM_ID]], 4
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    store <2 x i32> [[DATA]], ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret void
+;
+  %group.id = call i32 @llvm.amdgcn.workgroup.id.x()
+  %mask = icmp ult i32 %group.id, 4
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)
+  ret void
+}
+
+declare void @llvm.masked.store.v2i32.p0(<2 x i32>, ptr, i32, <2 x i1>)
+declare void @llvm.masked.store.v2f16.p0(<2 x half>, ptr, i32, <2 x i1>)
+declare void @llvm.masked.store.v2i32.p3(<2 x i32>, ptr addrspace(3), i32, <2 x i1>)
+declare noundef i32 @llvm.amdgcn.workitem.id.x()
+declare noundef i32 @llvm.amdgcn.workgroup.id.x()
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/lit.local.cfg b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000000..7c492428aec7614
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "AMDGPU" in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
index 8c95a630ebce767..fffb5f021e52d4c 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
@@ -32,8 +32,8 @@ define <2 x i64> @scalarize_v2i64(ptr %p, <2 x i1> %mask, <2 x i64> %passthru) {
 
 define <2 x i64> @scalarize_v2i64_ones_mask(ptr %p, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64_ones_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8
-; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+; CHECK-NEXT:    [[RET:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    ret <2 x i64> [[RET]]
 ;
   %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)
   ret <2 x i64> %ret
@@ -58,6 +58,25 @@ define <2 x i64> @scalarize_v2i64_const_mask(ptr %p, <2 x i64> %passthru) {
   ret <2 x i64> %ret
 }
 
+define <2 x i64> @scalarize_v2i64_splat_mask(ptr %p, i1 %mask, <2 x i64> %passthrough) {
+; CHECK-LABEL: @scalarize_v2i64_splat_mask(
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK:%.*]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label [[COND_LOAD:%.*]], label [[TMP1:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[RET_COND_LOAD:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    br label [[TMP1]]
+; CHECK:       1:
+; CHECK-NEXT:    [[RET:%.*]] = phi <2 x i64> [ [[RET_COND_LOAD]], [[COND_LOAD]] ], [ [[PASSTHROUGH:%.*]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RET]]
+;
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> %mask.splat, <2 x i64> %passthrough)
+  ret <2 x i64> %ret
+}
+
 ; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
 define <2 x i24> @scalarize_v2i24(ptr %p, <2 x i1> %mask, <2 x i24> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i24(
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
index f6e54bc6fe94d38..4e3679dc5da99e1 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
@@ -56,4 +56,22 @@ define void @scalarize_v2i64_const_mask(ptr %p, <2 x i64> %data) {
   ret void
 }
 
+define void @scalarize_v2i64_splat_mask(ptr %p, <2 x i64> %data, i1 %mask) {
+; CHECK-LABEL: @scalarize_v2i64_splat_mask(
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK:%.*]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label [[COND_STORE:%.*]], label [[TMP1:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    store <2 x i64> [[DATA:%.*]], ptr [[P:%.*]], align 8
+; CHECK-NEXT:    br label [[TMP1]]
+; CHECK:       1:
+; CHECK-NEXT:    ret void
+;
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.store.v2i64.p0(<2 x i64> %data, ptr %p, i32 8, <2 x i1> %mask.splat)
+  ret void
+}
+
 declare void @llvm.masked.store.v2i64.p0(<2 x i64>, ptr, i32, <2 x i1>)
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
index cb2bbb8e0b9317e..cd26d949836e681 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
@@ -838,15 +838,14 @@ define i32 @test_pr30188a(i1 zeroext %flag, i32 %x) {
 ; CHECK:       if.then:
 ; CHECK-NEXT:    call void @g()
 ; CHECK-NEXT:    [[ONE:%.*]] = load i32, ptr [[Y]], align 4
-; CHECK-NEXT:    [[TWO:%.*]] = add i32 [[ONE]], 2
-; CHECK-NEXT:    store i32 [[TWO]], ptr [[Y]], align 4
 ; CHECK-NEXT:    br label [[IF_END:%.*]]
 ; CHECK:       if.else:
 ; CHECK-NEXT:    [[THREE:%.*]] = load i32, ptr [[Z]], align 4
-; CHECK-NEXT:    [[FOUR:%.*]] = add i32 [[THREE]], 2
-; CHECK-NEXT:    store i32 [[FOUR]], ptr [[Y]], align 4
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
+; CHECK-NEXT:    [[THREE_SINK:%.*]] = phi i32 [ [[THREE]], [[IF_ELSE]] ], [ [[ONE]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[FOUR:%.*]] = add i32 [[THREE_SINK]], 2
+; CHECK-NEXT:    store i32 [[FOUR]], ptr [[Y]], align 4
 ; CHECK-NEXT:    ret i32 1
 ;
 entry:
@@ -914,15 +913,16 @@ define zeroext i1 @test_pr30244(i1 zeroext %flag, i1 zeroext %flag2, i32 %blksA,
 ; CHECK-NEXT:    br i1 [[FLAG:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[BLKSA:%.*]], [[NBLKS:%.*]]
-; CHECK-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[CMP]] to i8
-; CHECK-NEXT:    store i8 [[FROMBOOL1]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK-NEXT:    br label [[IF_END_SINK_SPLIT:%.*]]
 ; CHECK:       if.else:
-; CHECK-NEXT:    br i1 [[FLAG2:%.*]], label [[IF_THEN2:%.*]], label [[IF_END]]
+; CHECK-NEXT:    br i1 [[FLAG2:%.*]], label [[IF_THEN2:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.then2:
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[NBLKS]], [[BLKSB:%.*]]
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 [[ADD]], [[BLKSA]]
-; CHECK-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[CMP2]] to i8
+; CHECK-NEXT:    br label [[IF_END_SINK_SPLIT]]
+; CHECK:       if.end.sink.split:
+; CHECK-NEXT:    [[CMP2_SINK:%.*]] = phi i1 [ [[CMP2]], [[IF_THEN2]] ], [ [[CMP]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[CMP2_SINK]] to i8
 ; CHECK-NEXT:    store i8 [[FROMBOOL3]], ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll b/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
index 991b89a486e0f5a..e2705dc95ab9792 100644
--- a/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
@@ -154,3 +154,350 @@ exit:                                             ; preds = %bb2, %bb1, %bb0
   %result = phi i1 [ %0, %bb0 ], [ %1, %bb1 ], [ %2, %bb2 ]
   ret i1 %result
 }
+
+declare void @foo()
+
+define i1 @test_icmp_simple(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_simple(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i1 [ [[CMP1:%.*]], [[IF]] ], [ [[CMP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i1 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[CMP1]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP2]] = icmp ugt i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %cmp1 = icmp ult i32 %a, %b
+  call void @foo()
+  ret i1 %cmp1
+
+else:
+  %cmp2 = icmp ugt i32 %b, %a
+  call void @bar()
+  ret i1 %cmp2
+}
+
+define void @test_icmp_complex(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_complex(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF2:%.*]], label [[ELSE2:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[B]], [[A]]
+; CHECK-NEXT:    br i1 [[CMP2]], label [[IF2]], label [[ELSE2]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret void
+; CHECK:       if2:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else2:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %cmp1 = icmp ult i32 %a, %b
+  br i1 %cmp1, label %if2, label %else2
+
+else:
+  %cmp2 = icmp ugt i32 %b, %a
+  br i1 %cmp2, label %if2, label %else2
+
+if2:
+  call void @foo()
+  ret void
+
+else2:
+  call void @bar()
+  ret void
+}
+
+define i1 @test_icmp_wrong_operands(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_wrong_operands(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i1 [ [[CMP1:%.*]], [[IF]] ], [ [[CMP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i1 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[CMP1]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP2]] = icmp ugt i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %cmp1 = icmp ult i32 %a, %b
+  call void @foo()
+  ret i1 %cmp1
+
+else:
+  %cmp2 = icmp ugt i32 %a, %b
+  call void @bar()
+  ret i1 %cmp2
+}
+
+define i1 @test_icmp_wrong_pred(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_wrong_pred(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i1 [ [[CMP1:%.*]], [[IF]] ], [ [[CMP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i1 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[CMP1]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP2]] = icmp uge i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %cmp1 = icmp ult i32 %a, %b
+  call void @foo()
+  ret i1 %cmp1
+
+else:
+  %cmp2 = icmp uge i32 %b, %a
+  call void @bar()
+  ret i1 %cmp2
+}
+
+define i32 @test_binop(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_binop(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = add i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = add i32 %a, %b
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = add i32 %b, %a
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_binop_flags(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_binop_flags(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = add nuw nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = add nsw i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = add nuw nsw i32 %a, %b
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = add nsw i32 %b, %a
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_binop_not_commutative(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_binop_not_commutative(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = sub i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = sub i32 %a, %b
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = sub i32 %b, %a
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_binop_wrong_ops(i1 %c, i32 %a, i32 %b, i32 %d) {
+; CHECK-LABEL: @test_binop_wrong_ops(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = add i32 [[B]], [[D:%.*]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = add i32 %a, %b
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = add i32 %b, %d
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_intrin(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_intrin(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = call i32 @llvm.umin.i32(i32 [[B]], i32 [[A]])
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = call i32 @llvm.umin(i32 %a, i32 %b)
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = call i32 @llvm.umin(i32 %b, i32 %a)
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_intrin_not_same(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_intrin_not_same(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = call i32 @llvm.umax.i32(i32 [[B]], i32 [[A]])
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = call i32 @llvm.umin(i32 %a, i32 %b)
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = call i32 @llvm.umax(i32 %b, i32 %a)
+  call void @bar()
+  ret i32 %op2
+}
+
+define float @test_intrin_3arg(i1 %c, float %a, float %b, float %d) {
+; CHECK-LABEL: @test_intrin_3arg(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi float [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret float [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = call float @llvm.fma.f32(float [[A:%.*]], float [[B:%.*]], float [[D:%.*]])
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = call float @llvm.fma.f32(float [[B]], float [[A]], float [[D]])
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = call float @llvm.fma(float %a, float %b, float %d)
+  call void @foo()
+  ret float %op1
+
+else:
+  %op2 = call float @llvm.fma(float %b, float %a, float %d)
+  call void @bar()
+  ret float %op2
+}
+
+define float @test_intrin_3arg_wrong_args_commuted(i1 %c, float %a, float %b, float %d) {
+; CHECK-LABEL: @test_intrin_3arg_wrong_args_commuted(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi float [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret float [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = call float @llvm.fma.f32(float [[A:%.*]], float [[B:%.*]], float [[D:%.*]])
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = call float @llvm.fma.f32(float [[A]], float [[D]], float [[B]])
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = call float @llvm.fma(float %a, float %b, float %d)
+  call void @foo()
+  ret float %op1
+
+else:
+  %op2 = call float @llvm.fma(float %a, float %d, float %b)
+  call void @bar()
+  ret float %op2
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
new file mode 100644
index 000000000000000..4c93837f1422a9d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes='simplifycfg<sink-common-insts;switch-to-lookup>' < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+
+define void @pr104567(i8 %x, ptr %f) {
+; CHECK-LABEL: define void @pr104567(
+; CHECK-SAME: i8 [[X:%.*]], ptr [[F:%.*]]) {
+; CHECK-NEXT:  [[START:.*:]]
+; CHECK-NEXT:    [[Y:%.*]] = alloca [1 x i8], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[Y]])
+; CHECK-NEXT:    [[SWITCH_OFFSET:%.*]] = add nsw i8 [[X]], 4
+; CHECK-NEXT:    store i8 [[SWITCH_OFFSET]], ptr [[Y]], align 1
+; CHECK-NEXT:    call void [[F]](ptr [[Y]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[Y]])
+; CHECK-NEXT:    ret void
+;
+start:
+  %y = alloca [1 x i8], align 1
+  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %y)
+  switch i8 %x, label %default.unreachable [
+  i8 0, label %bb4
+  i8 1, label %bb3
+  i8 2, label %bb2
+  ]
+
+default.unreachable:
+  unreachable
+
+bb4:
+  store i8 4, ptr %y, align 1
+  br label %bb5
+
+bb3:
+  store i8 5, ptr %y, align 1
+  br label %bb5
+
+bb2:
+  store i8 6, ptr %y, align 1
+  br label %bb5
+
+bb5:
+  call void %f(ptr %y)
+  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %y)
+  ret void
+}
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views-bottleneck.s b/llvm/test/tools/llvm-mca/JSON/X86/views-bottleneck.s
new file mode 100644
index 000000000000000..c8ad60ee542093c
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views-bottleneck.s
@@ -0,0 +1,73 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=haswell --json --timeline-max-iterations=1 --bottleneck-analysis --resource-pressure=false --instruction-info=false < %s | FileCheck %s
+vaddps %xmm0, %xmm0, %xmm1
+vblendvps %xmm1, (%rdi), %xmm2, %xmm3
+
+# CHECK:      {
+# CHECK-NEXT:   "CodeRegions": [
+# CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 0,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 1,
+# CHECK-NEXT:             "ResourceOrRegID": 128,
+# CHECK-NEXT:             "ToID": 3,
+# CHECK-NEXT:             "Type": 3
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 3,
+# CHECK-NEXT:             "ResourceOrRegID": 128,
+# CHECK-NEXT:             "ToID": 5,
+# CHECK-NEXT:             "Type": 3
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 56,
+# CHECK-NEXT:         "RegisterDependencyCycles": 0,
+# CHECK-NEXT:         "ResourcePressure": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "HWPort5": 56
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "ResourcePressureCycles": 56,
+# CHECK-NEXT:         "TotalCycles": 209
+# CHECK-NEXT:       },
+# CHECK-NEXT:       "Instructions": [
+# CHECK-NEXT:         "vaddps\t%xmm0, %xmm0, %xmm1",
+# CHECK-NEXT:         "vblendvps\t%xmm1, (%rdi), %xmm2, %xmm3"
+# CHECK-NEXT:       ],
+# CHECK-NEXT:       "Name": "",
+# CHECK-NEXT:       "SummaryView": {
+# CHECK-NEXT:         "BlockRThroughput": 2,
+# CHECK-NEXT:         "DispatchWidth": 4,
+# CHECK-NEXT:         "IPC": 0.9569377990430622,
+# CHECK-NEXT:         "Instructions": 200,
+# CHECK-NEXT:         "Iterations": 100,
+# CHECK-NEXT:         "TotalCycles": 209,
+# CHECK-NEXT:         "TotaluOps": 400,
+# CHECK-NEXT:         "uOpsPerCycle": 1.9138755980861244
+# CHECK-NEXT:       }
+# CHECK-NEXT:     }
+# CHECK-NEXT:   ],
+# CHECK-NEXT:   "SimulationParameters": {
+# CHECK-NEXT:     "-march": "x86_64",
+# CHECK-NEXT:     "-mcpu": "haswell",
+# CHECK-NEXT:     "-mtriple": "x86_64-unknown-unknown"
+# CHECK-NEXT:   },
+# CHECK-NEXT:   "TargetInfo": {
+# CHECK-NEXT:     "CPUName": "haswell",
+# CHECK-NEXT:     "Resources": [
+# CHECK-NEXT:       "HWDivider",
+# CHECK-NEXT:       "HWFPDivider",
+# CHECK-NEXT:       "HWPort0",
+# CHECK-NEXT:       "HWPort1",
+# CHECK-NEXT:       "HWPort2",
+# CHECK-NEXT:       "HWPort3",
+# CHECK-NEXT:       "HWPort4",
+# CHECK-NEXT:       "HWPort5",
+# CHECK-NEXT:       "HWPort6",
+# CHECK-NEXT:       "HWPort7"
+# CHECK-NEXT:     ]
+# CHECK-NEXT:   }
+# CHECK-NEXT: }
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views-custom-parameters.s b/llvm/test/tools/llvm-mca/JSON/X86/views-custom-parameters.s
index 1301c4acda8321f..538175f6ac6155e 100644
--- a/llvm/test/tools/llvm-mca/JSON/X86/views-custom-parameters.s
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views-custom-parameters.s
@@ -16,6 +16,9 @@ add %edx, %edx
 # CHECK:      {
 # CHECK-NEXT:   "CodeRegions": [
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "PressureIncreaseCycles": 0
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-anonymous-regions.s b/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-anonymous-regions.s
index 8cd8d41a6ff65f3..ea9dbb632d7c1ca 100644
--- a/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-anonymous-regions.s
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-anonymous-regions.s
@@ -19,6 +19,29 @@ add %edx, %edx
 # CHECK:      {
 # CHECK-NEXT:   "CodeRegions": [
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 39,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 0,
+# CHECK-NEXT:             "ResourceOrRegID": 22,
+# CHECK-NEXT:             "ToID": 1,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 1,
+# CHECK-NEXT:             "ResourceOrRegID": 22,
+# CHECK-NEXT:             "ToID": 2,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 39,
+# CHECK-NEXT:         "RegisterDependencyCycles": 39,
+# CHECK-NEXT:         "ResourcePressure": [],
+# CHECK-NEXT:         "ResourcePressureCycles": 0,
+# CHECK-NEXT:         "TotalCycles": 103
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
@@ -175,6 +198,29 @@ add %edx, %edx
 # CHECK-NEXT:       }
 # CHECK-NEXT:     },
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 69,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 0,
+# CHECK-NEXT:             "ResourceOrRegID": 24,
+# CHECK-NEXT:             "ToID": 2,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 2,
+# CHECK-NEXT:             "ResourceOrRegID": 24,
+# CHECK-NEXT:             "ToID": 4,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 69,
+# CHECK-NEXT:         "RegisterDependencyCycles": 69,
+# CHECK-NEXT:         "ResourcePressure": [],
+# CHECK-NEXT:         "ResourcePressureCycles": 0,
+# CHECK-NEXT:         "TotalCycles": 103
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
@@ -411,6 +457,29 @@ add %edx, %edx
 # CHECK-NEXT:       }
 # CHECK-NEXT:     },
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 39,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 0,
+# CHECK-NEXT:             "ResourceOrRegID": 27,
+# CHECK-NEXT:             "ToID": 1,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 1,
+# CHECK-NEXT:             "ResourceOrRegID": 27,
+# CHECK-NEXT:             "ToID": 2,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 39,
+# CHECK-NEXT:         "RegisterDependencyCycles": 39,
+# CHECK-NEXT:         "ResourcePressure": [],
+# CHECK-NEXT:         "ResourcePressureCycles": 0,
+# CHECK-NEXT:         "TotalCycles": 103
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-region.s b/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-region.s
index 0b7bb16beeb9169..a000e9b7d21757b 100644
--- a/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-region.s
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-region.s
@@ -20,6 +20,9 @@ add %edx, %edx
 # CHECK:      {
 # CHECK-NEXT:   "CodeRegions": [
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "PressureIncreaseCycles": 0
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
@@ -164,6 +167,29 @@ add %edx, %edx
 # CHECK-NEXT:       }
 # CHECK-NEXT:     },
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 69,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 0,
+# CHECK-NEXT:             "ResourceOrRegID": 24,
+# CHECK-NEXT:             "ToID": 2,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 2,
+# CHECK-NEXT:             "ResourceOrRegID": 24,
+# CHECK-NEXT:             "ToID": 4,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 69,
+# CHECK-NEXT:         "RegisterDependencyCycles": 69,
+# CHECK-NEXT:         "ResourcePressure": [],
+# CHECK-NEXT:         "ResourcePressureCycles": 0,
+# CHECK-NEXT:         "TotalCycles": 103
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views.s b/llvm/test/tools/llvm-mca/JSON/X86/views.s
index df85f2203f314bc..324d7b8efa07a2b 100644
--- a/llvm/test/tools/llvm-mca/JSON/X86/views.s
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views.s
@@ -16,6 +16,9 @@ add %edx, %edx
 # CHECK:      {
 # CHECK-NEXT:   "CodeRegions": [
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "PressureIncreaseCycles": 0
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/load.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/load.s
index 3cf207b594ad996..216d2f6c9b57120 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/load.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/load.s
@@ -42,15 +42,20 @@ fld ft0, 0(a0)
 # CHECK-NEXT: [5]   - SiFiveP400IEXQ2
 # CHECK-NEXT: [6]   - SiFiveP400Load
 # CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]
-# CHECK-NEXT:  -      -      -      -      -      -     5.00    -
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -      -      -     5.00    -      -      -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     lw	t0, 0(a0)
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     ld	t0, 0(a0)
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     flh	ft0, 0(a0)
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     flw	ft0, 0(a0)
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     fld	ft0, 0(a0)
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     lw	t0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     ld	t0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     flh	ft0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     flw	ft0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     fld	ft0, 0(a0)
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vislide-vx.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vislide-vx.s
new file mode 100644
index 000000000000000..c4934f4ab02095a
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vislide-vx.s
@@ -0,0 +1,108 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e32, m2, tu, mu
+vslidedown.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m4, tu, mu
+vslidedown.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m8, tu, mu
+vslidedown.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m2, tu, mu
+vslideup.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m4, tu, mu
+vslideup.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m8, tu, mu
+vslideup.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m2, tu, mu
+vslideup.vx v5, v7, x6, v0.t
+
+vsetvli zero, zero, e32, m4, tu, mu
+vslideup.vx v5, v7, x6, v0.t
+
+vsetvli zero, zero, e32, m8, tu, mu
+vslideup.vx v5, v7, x6, v0.t
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      18
+# CHECK-NEXT: Total Cycles:      125
+# CHECK-NEXT: Total uOps:        18
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.14
+# CHECK-NEXT: IPC:               0.14
+# CHECK-NEXT: Block RThroughput: 121.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      11    11.00                       vslidedown.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      14    14.00                       vslidedown.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      20    20.00                       vslidedown.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      10    10.00                       vslideup.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      12    12.00                       vslideup.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      16    16.00                       vslideup.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      10    10.00                       vslideup.vx v5, v7, t1, v0.t
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      12    12.00                       vslideup.vx v5, v7, t1, v0.t
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      16    16.00                       vslideup.vx v5, v7, t1, v0.t
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     9.00    -      -      -      -     121.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     11.00   -      -      -     vslidedown.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     14.00   -      -      -     vslidedown.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     20.00   -      -      -     vslidedown.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vslideup.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vslideup.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vslideup.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vslideup.vx  v5, v7, t1, v0.t
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vslideup.vx  v5, v7, t1, v0.t
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vslideup.vx  v5, v7, t1, v0.t
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s
new file mode 100644
index 000000000000000..aba6b2f3e24a5b7
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s
@@ -0,0 +1,9827 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg2e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg3e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg4e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg5e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg6e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg7e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg8e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vsseg2e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vsseg3e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vsseg4e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg5e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg6e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg7e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg8e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vssseg2e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vssseg3e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vssseg4e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg5e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg6e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg7e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg8e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg2e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg3e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg4e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg5e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg6e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg7e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg8e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      3240
+# CHECK-NEXT: Total Cycles:      217825
+# CHECK-NEXT: Total uOps:        3240
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.01
+# CHECK-NEXT: IPC:               0.01
+# CHECK-NEXT: Block RThroughput: 124596.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei64.v  v8, (a0), v16
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     1620.00  -     -      -      -      -      -     124596.00 96850.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei64.v v8, (a0), v16
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s
new file mode 100644
index 000000000000000..31178e8e238f1ad
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s
@@ -0,0 +1,895 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, mf8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, mf8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, mf8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m8, tu, mu
+vmv8r.v v8, v16
+
+vsetvli zero, zero, e64, m1, tu, mu
+vmv.s.x v8, x5
+vmv.x.s x7, v16
+
+vsetvli zero, zero, e64, m2, tu, mu
+vmv.s.x v8, x5
+vmv.x.s x7, v16
+
+vsetvli zero, zero, e64, m4, tu, mu
+vmv.s.x v8, x5
+vmv.x.s x7, v16
+
+vsetvli zero, zero, e64, m8, tu, mu
+vmv.s.x v8, x5
+vmv.x.s x7, v16
+
+vsetvli zero, zero, e64, m1, tu, mu
+vfmv.s.f v8, f5
+vfmv.f.s f7, v16
+
+vsetvli zero, zero, e64, m2, tu, mu
+vfmv.s.f v8, f5
+vfmv.f.s f7, v16
+
+vsetvli zero, zero, e64, m4, tu, mu
+vfmv.s.f v8, f5
+vfmv.f.s f7, v16
+
+vsetvli zero, zero, e64, m8, tu, mu
+vfmv.s.f v8, f5
+vfmv.f.s f7, v16
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      280
+# CHECK-NEXT: Total Cycles:      523
+# CHECK-NEXT: Total uOps:        280
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.54
+# CHECK-NEXT: IPC:               0.54
+# CHECK-NEXT: Block RThroughput: 512.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv.s.x v8, t0
+# CHECK-NEXT:  1      2     2.00                        vmv.x.s t2, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv.s.x v8, t0
+# CHECK-NEXT:  1      2     2.00                        vmv.x.s t2, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv.s.x v8, t0
+# CHECK-NEXT:  1      2     2.00                        vmv.x.s t2, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv.s.x v8, t0
+# CHECK-NEXT:  1      2     2.00                        vmv.x.s t2, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vfmv.s.f  v8, ft5
+# CHECK-NEXT:  1      6     2.00                        vfmv.f.s  ft7, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vfmv.s.f  v8, ft5
+# CHECK-NEXT:  1      6     2.00                        vfmv.f.s  ft7, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vfmv.s.f  v8, ft5
+# CHECK-NEXT:  1      6     2.00                        vfmv.f.s  ft7, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vfmv.s.f  v8, ft5
+# CHECK-NEXT:  1      6     2.00                        vfmv.f.s  ft7, v16
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     136.00  -      -      -      -     512.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.s.x  v8, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.x.s  t2, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.s.x  v8, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.x.s  t2, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.s.x  v8, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.x.s  t2, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.s.x  v8, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.x.s  t2, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.s.f v8, ft5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.f.s ft7, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.s.f v8, ft5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.f.s ft7, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.s.f v8, ft5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.f.s ft7, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.s.f v8, ft5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.f.s ft7, v16
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vreduce.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vreduce.s
new file mode 100644
index 000000000000000..0fc0bf03d7ecb27
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vreduce.s
@@ -0,0 +1,438 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# Simple integer reductions: varies by LMUL
+vsetvli zero, zero, e32, m1, tu, mu
+vredsum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vredsum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vredsum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vredsum.vs v5, v7, v8
+
+# Advanced integer reductions: varies by LMUL and SEW
+vsetvli zero, zero, e16, m1, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m2, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m4, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m8, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m1, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m2, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m4, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m8, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+# Simple floating point reductions: varies by LMUL and SEW
+vsetvli zero, zero, e16, m1, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m2, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m4, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m8, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m1, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m2, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m4, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m8, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+# Advanced floating point reductions: varies by LMUL and SEW
+vsetvli zero, zero, e16, m1, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m2, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m4, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m8, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m1, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m2, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m4, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m8, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      116
+# CHECK-NEXT: Total Cycles:      921
+# CHECK-NEXT: Total uOps:        116
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.13
+# CHECK-NEXT: IPC:               0.13
+# CHECK-NEXT: Block RThroughput: 916.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vredsum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00                        vredsum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vredsum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      16    9.00                        vredsum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      8     4.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      10    5.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      10    5.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      12    8.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      12    8.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      15    14.00                       vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      15    14.00                       vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      8     4.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      10    6.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      10    6.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      13    12.00                       vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      13    12.00                       vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      4     2.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      8     5.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      8     5.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      11    10.00                       vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      11    10.00                       vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      16    16.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      22    16.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      32    32.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      28    16.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      64    64.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      34    16.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      128   128.00                      vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      10    10.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      10    10.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      16    10.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      20    20.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      22    10.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      40    40.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      28    10.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      80    80.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      6     6.00                        vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      6     6.00                        vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      12    6.00                        vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      12    12.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      18    6.00                        vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      24    24.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      24    6.00                        vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      48    48.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      8     4.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      10    6.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      10    6.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      13    12.00                       vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      13    12.00                       vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      8     4.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      10    6.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      10    6.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      13    12.00                       vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      13    12.00                       vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      4     2.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      8     5.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      8     5.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      11    10.00                       vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      11    10.00                       vfredmax.vs v5, v7, v8
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     40.00   -      -      -      -     916.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vredsum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vredsum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredsum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     9.00    -      -      -     vredsum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     14.00   -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     14.00   -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     32.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     64.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     128.00  -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     20.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     40.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     80.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     24.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     48.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredmax.vs  v5, v7, v8
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vrgather.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vrgather.s
new file mode 100644
index 000000000000000..ef248509571931a
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vrgather.s
@@ -0,0 +1,86 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e32, m1, tu, mu
+vrgather.vv  v5, v7, v8
+vcompress.vm v4, v9, v0
+
+vsetvli zero, zero, e32, m2, tu, mu
+vrgather.vv  v5, v7, v8
+vcompress.vm v4, v9, v0
+
+vsetvli zero, zero, e32, m4, tu, mu
+vrgather.vv  v5, v7, v8
+vcompress.vm v4, v9, v0
+
+vsetvli zero, zero, e32, m8, tu, mu
+vrgather.vv  v5, v7, v8
+vcompress.vm v4, v9, v0
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      12
+# CHECK-NEXT: Total Cycles:      92
+# CHECK-NEXT: Total uOps:        12
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.13
+# CHECK-NEXT: IPC:               0.13
+# CHECK-NEXT: Block RThroughput: 106.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vrgather.vv v5, v7, v8
+# CHECK-NEXT:  1      3     1.00                        vcompress.vm  v4, v9, v0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      6     12.00                       vrgather.vv v5, v7, v8
+# CHECK-NEXT:  1      6     12.00                       vcompress.vm  v4, v9, v0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      6     16.00                       vrgather.vv v5, v7, v8
+# CHECK-NEXT:  1      6     16.00                       vcompress.vm  v4, v9, v0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      6     24.00                       vrgather.vv v5, v7, v8
+# CHECK-NEXT:  1      6     24.00                       vcompress.vm  v4, v9, v0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     4.00    -      -      -      -     106.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrgather.vv  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcompress.vm v4, v9, v0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vrgather.vv  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vcompress.vm v4, v9, v0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vrgather.vv  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vcompress.vm v4, v9, v0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     24.00   -      -      -     vrgather.vv  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     24.00   -      -      -     vcompress.vm v4, v9, v0
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vshift-vmul.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vshift-vmul.s
new file mode 100644
index 000000000000000..7a0cf27f37425df
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vshift-vmul.s
@@ -0,0 +1,132 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e32, m1, ta, ma
+
+vsll.vv v1, v2, v5
+vsll.vx v1, v2, t0
+vsll.vi v1, v2, 7
+
+vsrl.vv v1, v2, v5
+vsrl.vx v1, v2, t0
+vsrl.vi v1, v2, 7
+
+vsra.vv v1, v2, v5
+vsra.vx v1, v2, t0
+vsra.vi v1, v2, 7
+
+vsetvli zero, zero, e32, mf4, ta, ma
+
+vsll.vv v1, v2, v5
+vsll.vx v1, v2, t0
+vsll.vi v1, v2, 7
+
+vsrl.vv v1, v2, v5
+vsrl.vx v1, v2, t0
+vsrl.vi v1, v2, 7
+
+vsra.vv v1, v2, v5
+vsra.vx v1, v2, t0
+vsra.vi v1, v2, 7
+
+vsetvli zero, zero, e32, m8, ta, ma
+
+vmul.vv v1, v2, v5
+vmul.vx v1, v2, t1
+
+vmadd.vv v1, v2, v5
+vmadd.vx v1, t1, v2
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      25
+# CHECK-NEXT: Total Cycles:      57
+# CHECK-NEXT: Total uOps:        25
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.44
+# CHECK-NEXT: IPC:               0.44
+# CHECK-NEXT: Block RThroughput: 50.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, ta, ma
+# CHECK-NEXT:  1      2     1.00                        vsll.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsll.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsll.vi v1, v2, 7
+# CHECK-NEXT:  1      2     1.00                        vsrl.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsrl.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsrl.vi v1, v2, 7
+# CHECK-NEXT:  1      2     1.00                        vsra.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsra.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsra.vi v1, v2, 7
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, ta, ma
+# CHECK-NEXT:  1      2     1.00                        vsll.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsll.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsll.vi v1, v2, 7
+# CHECK-NEXT:  1      2     1.00                        vsrl.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsrl.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsrl.vi v1, v2, 7
+# CHECK-NEXT:  1      2     1.00                        vsra.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsra.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsra.vi v1, v2, 7
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, ta, ma
+# CHECK-NEXT:  1      9     8.00                        vmul.vv v1, v2, v5
+# CHECK-NEXT:  1      9     8.00                        vmul.vx v1, v2, t1
+# CHECK-NEXT:  1      9     8.00                        vmadd.vv  v1, v2, v5
+# CHECK-NEXT:  1      9     8.00                        vmadd.vx  v1, t1, v2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     3.00    -      -      -      -     50.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, ta, ma
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, ta, ma
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, ta, ma
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmul.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmul.vx  v1, v2, t1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmadd.vv v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmadd.vx v1, t1, v2
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbb.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbb.s
new file mode 100644
index 000000000000000..236c8f0199f6633
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbb.s
@@ -0,0 +1,460 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e16, mf4, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v8, v4, v12
+vwsll.vx v8, v4, a0
+vwsll.vi v8, v4, 8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vandn.vv v8, v16, v24
+vandn.vx v8, v16, a0
+vbrev.v  v8, v16
+vbrev8.v v8, v16
+vrev8.v  v8, v16
+vclz.v   v8, v16
+vctz.v   v8, v16
+vcpop.v  v8, v16
+vrol.vv  v8, v16, v24
+vrol.vx  v8, v16, a0
+vror.vv  v8, v16, v24
+vror.vx  v8, v16, a0
+vror.vi  v8, v16, 8
+
+# Show SEW does not matter
+vsetvli zero, zero, e16, m4, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v  v4, v8
+vbrev8.v v4, v8
+vrev8.v  v4, v8
+vclz.v   v4, v8
+vctz.v   v4, v8
+vcpop.v  v4, v8
+vrol.vv  v4, v8, v12
+vrol.vx  v4, v8, a0
+vror.vv  v4, v8, v12
+vror.vx  v4, v8, a0
+vror.vi  v4, v8, 8
+vwsll.vv v8, v4, v12
+vwsll.vx v8, v4, a0
+vwsll.vi v8, v4, 8
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      133
+# CHECK-NEXT: Total Cycles:      330
+# CHECK-NEXT: Total uOps:        133
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.40
+# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: Block RThroughput: 328.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     2.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     2.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     2.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     2.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     2.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     2.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     2.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     4.00                        vwsll.vv  v8, v4, v12
+# CHECK-NEXT:  1      2     4.00                        vwsll.vx  v8, v4, a0
+# CHECK-NEXT:  1      2     4.00                        vwsll.vi  v8, v4, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vandn.vv  v8, v16, v24
+# CHECK-NEXT:  1      2     8.00                        vandn.vx  v8, v16, a0
+# CHECK-NEXT:  1      2     8.00                        vbrev.v v8, v16
+# CHECK-NEXT:  1      2     8.00                        vbrev8.v  v8, v16
+# CHECK-NEXT:  1      2     8.00                        vrev8.v v8, v16
+# CHECK-NEXT:  1      2     8.00                        vclz.v  v8, v16
+# CHECK-NEXT:  1      2     8.00                        vctz.v  v8, v16
+# CHECK-NEXT:  1      2     8.00                        vcpop.v v8, v16
+# CHECK-NEXT:  1      2     8.00                        vrol.vv v8, v16, v24
+# CHECK-NEXT:  1      2     8.00                        vrol.vx v8, v16, a0
+# CHECK-NEXT:  1      2     8.00                        vror.vv v8, v16, v24
+# CHECK-NEXT:  1      2     8.00                        vror.vx v8, v16, a0
+# CHECK-NEXT:  1      2     8.00                        vror.vi v8, v16, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     4.00                        vwsll.vv  v8, v4, v12
+# CHECK-NEXT:  1      2     4.00                        vwsll.vx  v8, v4, a0
+# CHECK-NEXT:  1      2     4.00                        vwsll.vi  v8, v4, 8
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     8.00    -      -      -      -     328.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vv v8, v4, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vx v8, v4, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vi v8, v4, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vandn.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vandn.vx v8, v16, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vbrev.v  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vbrev8.v v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vrev8.v  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclz.v v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vctz.v v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vcpop.v  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vrol.vv  v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vrol.vx  v8, v16, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vror.vv  v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vror.vx  v8, v16, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vror.vi  v8, v16, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vv v8, v4, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vx v8, v4, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vi v8, v4, 8
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbc.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbc.s
new file mode 100644
index 000000000000000..1e304f6b75a140f
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbc.s
@@ -0,0 +1,112 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# These instructions only work with e64
+
+vsetvli zero, zero, e64, m1, tu, mu
+vclmul.vv v4, v8, v12
+vclmul.vx v4, v8, a0
+vclmulh.vv v4, v8, v12
+vclmulh.vx v4, v8, a0
+
+vsetvli zero, zero, e64, m2, tu, mu
+vclmul.vv v4, v8, v12
+vclmul.vx v4, v8, a0
+vclmulh.vv v4, v8, v12
+vclmulh.vx v4, v8, a0
+
+vsetvli zero, zero, e64, m4, tu, mu
+vclmul.vv v4, v8, v12
+vclmul.vx v4, v8, a0
+vclmulh.vv v4, v8, v12
+vclmulh.vx v4, v8, a0
+
+vsetvli zero, zero, e64, m8, tu, mu
+vclmul.vv  v8, v12, v24
+vclmul.vx  v8, v12, a0
+vclmulh.vv v8, v12, v24
+vclmulh.vx v8, v12, a0
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      20
+# CHECK-NEXT: Total Cycles:      58
+# CHECK-NEXT: Total uOps:        20
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.34
+# CHECK-NEXT: IPC:               0.34
+# CHECK-NEXT: Block RThroughput: 60.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vclmul.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vclmul.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vclmulh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vclmulh.vx  v4, v8, a0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vclmul.vv v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vclmul.vx v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vclmulh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vclmulh.vx  v4, v8, a0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vclmul.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vclmul.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vclmulh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vclmulh.vx  v4, v8, a0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vclmul.vv v8, v12, v24
+# CHECK-NEXT:  1      2     8.00                        vclmul.vx v8, v12, a0
+# CHECK-NEXT:  1      2     8.00                        vclmulh.vv  v8, v12, v24
+# CHECK-NEXT:  1      2     8.00                        vclmulh.vx  v8, v12, a0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     4.00    -      -      -      -     60.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclmul.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclmul.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclmulh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclmulh.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclmul.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclmul.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclmulh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclmulh.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclmul.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclmul.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclmulh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclmulh.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclmul.vv  v8, v12, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclmul.vx  v8, v12, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclmulh.vv v8, v12, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclmulh.vx v8, v12, a0
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkg.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkg.s
new file mode 100644
index 000000000000000..8e498a678138825
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkg.s
@@ -0,0 +1,127 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e16, mf4, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vghsh.vv v8, v16, v24
+vgmul.vv v8, v16
+
+# Show SEW does not matter
+vsetvli zero, zero, e64, m4, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      24
+# CHECK-NEXT: Total Cycles:      76
+# CHECK-NEXT: Total uOps:        24
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.32
+# CHECK-NEXT: IPC:               0.32
+# CHECK-NEXT: Block RThroughput: 72.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     8.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     8.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vghsh.vv  v8, v16, v24
+# CHECK-NEXT:  1      2     8.00                        vgmul.vv  v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vgmul.vv  v4, v8
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     8.00    -      -      -      -     72.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vghsh.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vgmul.vv v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vgmul.vv v4, v8
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkned.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkned.s
new file mode 100644
index 000000000000000..791069b87eee15c
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkned.s
@@ -0,0 +1,203 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# These instructions only support e32
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vaesef.vv v4, v8
+vaesef.vs v4, v8
+vaesem.vv v4, v8
+vaesem.vs v4, v8
+vaesdm.vv v4, v8
+vaesdm.vs v4, v8
+vaeskf1.vi v4, v8, 8
+vaeskf2.vi v4, v8, 8
+vaesz.vs v4, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vaesef.vv v4, v8
+vaesef.vs v4, v8
+vaesem.vv v4, v8
+vaesem.vs v4, v8
+vaesdm.vv v4, v8
+vaesdm.vs v4, v8
+vaeskf1.vi v4, v8, 8
+vaeskf2.vi v4, v8, 8
+vaesz.vs v4, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vaesef.vv v4, v8
+vaesef.vs v4, v8
+vaesem.vv v4, v8
+vaesem.vs v4, v8
+vaesdm.vv v4, v8
+vaesdm.vs v4, v8
+vaeskf1.vi v4, v8, 8
+vaeskf2.vi v4, v8, 8
+vaesz.vs v4, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vaesef.vv v4, v8
+vaesef.vs v4, v8
+vaesem.vv v4, v8
+vaesem.vs v4, v8
+vaesdm.vv v4, v8
+vaesdm.vs v4, v8
+vaeskf1.vi v4, v8, 8
+vaeskf2.vi v4, v8, 8
+vaesz.vs v4, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vaesef.vv  v8, v16
+vaesef.vs  v8, v16
+vaesem.vv  v8, v16
+vaesem.vs  v8, v16
+vaesdm.vv  v8, v16
+vaesdm.vs  v8, v16
+vaeskf1.vi v8, v16, 8
+vaeskf2.vi v8, v16, 8
+vaesz.vs   v8, v16
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      50
+# CHECK-NEXT: Total Cycles:      142
+# CHECK-NEXT: Total uOps:        50
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.35
+# CHECK-NEXT: IPC:               0.35
+# CHECK-NEXT: Block RThroughput: 144.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vaesef.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesef.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesem.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesem.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesdm.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesdm.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaeskf1.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vaeskf2.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vaesz.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vaesef.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesef.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesem.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesem.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesdm.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesdm.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaeskf1.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vaeskf2.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vaesz.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vaesef.vv v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesef.vs v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesem.vv v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesem.vs v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesdm.vv v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesdm.vs v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaeskf1.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     2.00                        vaeskf2.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     2.00                        vaesz.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vaesef.vv v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesef.vs v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesem.vv v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesem.vs v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesdm.vv v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesdm.vs v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaeskf1.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     4.00                        vaeskf2.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     4.00                        vaesz.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vaesef.vv v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesef.vs v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesem.vv v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesem.vs v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesdm.vv v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesdm.vs v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaeskf1.vi  v8, v16, 8
+# CHECK-NEXT:  1      2     8.00                        vaeskf2.vi  v8, v16, 8
+# CHECK-NEXT:  1      2     8.00                        vaesz.vs  v8, v16
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     5.00    -      -      -      -     144.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesef.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesef.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesem.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesem.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesdm.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesdm.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaeskf1.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaeskf2.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesz.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesef.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesef.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesem.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesem.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesdm.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesdm.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaeskf1.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaeskf2.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesz.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesef.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesef.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesem.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesem.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesdm.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesdm.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaeskf1.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaeskf2.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesz.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesef.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesef.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesem.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesem.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesdm.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesdm.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaeskf1.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaeskf2.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesz.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesef.vv  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesef.vs  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesem.vv  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesem.vs  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesdm.vv  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesdm.vs  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaeskf1.vi v8, v16, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaeskf2.vi v8, v16, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesz.vs v8, v16
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvknhb.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvknhb.s
new file mode 100644
index 000000000000000..285d21cbd33d53f
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvknhb.s
@@ -0,0 +1,152 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# SEW is only e32 or e64
+
+vsetvli zero, zero, e32, m1, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e32, m2, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e32, m4, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e32, m8, tu, mu
+vsha2ms.vv v8, v16, v24
+vsha2ch.vv v8, v16, v24
+vsha2cl.vv v8, v16, v24
+
+vsetvli zero, zero, e64, m1, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e64, m2, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e64, m4, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e64, m8, tu, mu
+vsha2ms.vv v8, v16, v24
+vsha2ch.vv v8, v16, v24
+vsha2cl.vv v8, v16, v24
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      32
+# CHECK-NEXT: Total Cycles:      102
+# CHECK-NEXT: Total uOps:        32
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 90.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      3     2.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      3     4.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     4.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     4.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      3     8.00                        vsha2ms.vv  v8, v16, v24
+# CHECK-NEXT:  1      3     8.00                        vsha2ch.vv  v8, v16, v24
+# CHECK-NEXT:  1      3     8.00                        vsha2cl.vv  v8, v16, v24
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      3     2.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      3     4.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     4.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     4.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      3     8.00                        vsha2ms.vv  v8, v16, v24
+# CHECK-NEXT:  1      3     8.00                        vsha2ch.vv  v8, v16, v24
+# CHECK-NEXT:  1      3     8.00                        vsha2cl.vv  v8, v16, v24
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     8.00    -      -      -      -     90.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2ms.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2ch.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2cl.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2ms.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2ch.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2cl.vv v8, v16, v24
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksed.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksed.s
new file mode 100644
index 000000000000000..127998f34aa38a7
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksed.s
@@ -0,0 +1,113 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# These instructions only support e32
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vsm4k.vi v4, v8, 8
+vsm4r.vv v4, v8
+vsm4r.vs v4, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vsm4k.vi v4, v8, 8
+vsm4r.vv v4, v8
+vsm4r.vs v4, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vsm4k.vi v4, v8, 8
+vsm4r.vv v4, v8
+vsm4r.vs v4, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vsm4k.vi v4, v8, 8
+vsm4r.vv v4, v8
+vsm4r.vs v4, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vsm4k.vi v8, v16, 8
+vsm4r.vv v8, v16
+vsm4r.vs v8, v16
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      20
+# CHECK-NEXT: Total Cycles:      47
+# CHECK-NEXT: Total uOps:        20
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.43
+# CHECK-NEXT: IPC:               0.43
+# CHECK-NEXT: Block RThroughput: 48.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vsm4k.vi  v4, v8, 8
+# CHECK-NEXT:  1      3     1.00                        vsm4r.vv  v4, v8
+# CHECK-NEXT:  1      3     1.00                        vsm4r.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vsm4k.vi  v4, v8, 8
+# CHECK-NEXT:  1      3     1.00                        vsm4r.vv  v4, v8
+# CHECK-NEXT:  1      3     1.00                        vsm4r.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      3     2.00                        vsm4k.vi  v4, v8, 8
+# CHECK-NEXT:  1      3     2.00                        vsm4r.vv  v4, v8
+# CHECK-NEXT:  1      3     2.00                        vsm4r.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      3     4.00                        vsm4k.vi  v4, v8, 8
+# CHECK-NEXT:  1      3     4.00                        vsm4r.vv  v4, v8
+# CHECK-NEXT:  1      3     4.00                        vsm4r.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      3     8.00                        vsm4k.vi  v8, v16, 8
+# CHECK-NEXT:  1      3     8.00                        vsm4r.vv  v8, v16
+# CHECK-NEXT:  1      3     8.00                        vsm4r.vs  v8, v16
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     5.00    -      -      -      -     48.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4k.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4r.vv v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4r.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4k.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4r.vv v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4r.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm4k.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm4r.vv v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm4r.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm4k.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm4r.vv v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm4r.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsm4k.vi v8, v16, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsm4r.vv v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsm4r.vs v8, v16
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksh.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksh.s
new file mode 100644
index 000000000000000..4c7b4dce8b6f486
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksh.s
@@ -0,0 +1,98 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# These instructions only support e32
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vsm3me.vv v4, v8, v12
+vsm3c.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vsm3me.vv v4, v8, v12
+vsm3c.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vsm3me.vv v4, v8, v12
+vsm3c.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vsm3me.vv v4, v8, v12
+vsm3c.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vsm3me.vv v8, v16, v24
+vsm3c.vi  v8, v16, 8
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      15
+# CHECK-NEXT: Total Cycles:      28
+# CHECK-NEXT: Total uOps:        15
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.54
+# CHECK-NEXT: IPC:               0.54
+# CHECK-NEXT: Block RThroughput: 25.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      6     1.00                        vsm3me.vv v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsm3c.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      6     1.00                        vsm3me.vv v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsm3c.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vsm3me.vv v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsm3c.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      6     4.00                        vsm3me.vv v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsm3c.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      6     8.00                        vsm3me.vv v8, v16, v24
+# CHECK-NEXT:  1      3     4.00                        vsm3c.vi  v8, v16, 8
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     5.00    -      -      -      -     25.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3me.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3c.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3me.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3c.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm3me.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3c.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm3me.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm3c.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsm3me.vv  v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm3c.vi v8, v16, 8
+
diff --git a/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx1.s
index 3914f865089247f..00e69d8960565fe 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx1.s
@@ -1429,8 +1429,8 @@ vzeroupper
 # CHECK-NEXT:  1      100   0.25    *                   vpcmpistri	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      100   0.25                        vpcmpistrm	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      100   0.25    *                   vpcmpistrm	$1, (%rax), %xmm2
-# CHECK-NEXT:  1      100   0.25                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  8      3     3.00                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  12     8     3.00    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      8     0.50    *                   vpermilpd	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpermilpd	%xmm0, %xmm1, %xmm2
@@ -1738,7 +1738,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT: 175.00 175.00  -      -      -      -      -     144.25 227.25 223.75 315.75  -
+# CHECK-NEXT: 175.50 175.50  -      -      -      -      -     150.25 230.25 226.75 315.75  -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -2141,8 +2141,8 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistri	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, (%rax), %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vperm2f128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     3.00   1.50   1.50    -      -     vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     3.00   1.50   1.50    -      -     vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
index 93b601e9285f162..a3ebe63e7189868 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
@@ -558,8 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      8     1.00    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      1     0.67                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.67    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  8      3     3.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  12     8     3.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      2     1.00                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      9     1.00    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      2     1.00                        vpermpd	$1, %ymm0, %ymm2
@@ -778,7 +778,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT: 66.50  66.50   -      -      -      -      -     120.17 239.17 158.00 66.67   -
+# CHECK-NEXT: 67.00  67.00   -      -      -      -      -     126.17 242.17 161.00 66.67   -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -880,8 +880,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -     1.00    -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67    -     0.67    -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.67   0.67    -     0.67    -     vpcmpgtw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     3.00   1.50   1.50    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     3.00   1.50   1.50    -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -     vpermpd	$1, %ymm0, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx1.s
index 37e2cf4500c38a5..96ae0575969671d 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx1.s
@@ -1429,8 +1429,8 @@ vzeroupper
 # CHECK-NEXT:  1      100   0.25    *                   vpcmpistri	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      100   0.25                        vpcmpistrm	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      100   0.25    *                   vpcmpistrm	$1, (%rax), %xmm2
-# CHECK-NEXT:  1      100   0.25                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.00                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     1.00    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      8     0.50    *                   vpermilpd	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      3     0.50                        vpermilpd	%xmm0, %xmm1, %xmm2
@@ -1739,7 +1739,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
-# CHECK-NEXT: 117.00 117.00 117.00 0.25   0.25   0.25   0.25    -     126.58 192.58 196.75 305.08  -
+# CHECK-NEXT: 117.33 117.33 117.33 0.25   0.25   0.25   0.25    -     126.58 192.58 198.75 305.08  -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
@@ -2142,8 +2142,8 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistri	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, (%rax), %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vperm2f128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -     vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -     1.00    -      -     vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx2.s
index 33d18b0fa0fb551..7be45be2c3578e7 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx2.s
@@ -558,8 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.33                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  1      8     0.33    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     1.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      2     0.50                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  1      9     0.50    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      2     0.50                        vpermpd	$1, %ymm0, %ymm2
@@ -779,7 +779,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
-# CHECK-NEXT: 44.33  44.33  44.33   -      -      -      -      -     63.17  125.67 71.00  34.17   -
+# CHECK-NEXT: 44.67  44.67  44.67   -      -      -      -      -     63.17  125.67 73.00  34.17   -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
@@ -881,8 +881,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -     0.50    -      -     0.50    -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33    -     0.33    -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -     0.33   0.33    -     0.33    -     vpcmpgtw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -     1.00    -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50    -      -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermpd	$1, %ymm0, %ymm2
diff --git a/llvm/test/tools/llvm-objcopy/wasm/basic-only-section.test b/llvm/test/tools/llvm-objcopy/wasm/basic-only-section.test
index fe47d5c027733a8..f30743c632bc863 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/basic-only-section.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/basic-only-section.test
@@ -1,15 +1,15 @@
 ## Test --only-section.
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy --only-section=foo %t %t2
-# RUN: obj2yaml %t2 | FileCheck --implicit-check-not TYPE --implicit-check-not linking %s
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not TYPE --implicit-check-not producers %s
 
 ## Test that it's the same with only-section + keep-section (for the same section).
 # RUN: llvm-objcopy --only-section=foo --keep-section=foo %t %t2
-# RUN: obj2yaml %t2 | FileCheck --implicit-check-not TYPE --implicit-check-not linking %s
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not TYPE --implicit-check-not producers %s
 
 ## Also test that only-section overrides remove-section.
 # RUN: llvm-objcopy --only-section=foo --remove-section=foo %t %t2
-# RUN: obj2yaml %t2 | FileCheck --implicit-check-not linking %s
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not producers %s
 
 ## This file has both known and custom sections. Check that only the foo section is left.
 # CHECK:      Sections:
@@ -19,10 +19,10 @@
 # CHECK-NEXT: ...
 
 ## Test that only-section + keep-section keeps both sections.
-# RUN: llvm-objcopy --only-section=foo --keep-section=linking %t %t2
+# RUN: llvm-objcopy --only-section=foo --keep-section=producers %t %t2
 # RUN: obj2yaml %t2 | FileCheck --implicit-check-not=TYPE --check-prefix=KEEP %s
 # KEEP: Name: foo
-# KEEP: Name: linking
+# KEEP: Name: producers
 
 --- !WASM
 FileHeader:
@@ -39,5 +39,7 @@ Sections:
         ReturnTypes:
           - F32
   - Type: CUSTOM
-    Name: linking
-    Version: 2
+    Name: producers
+    Tools:
+      - Name:   clang
+        Version: 9.0.0
diff --git a/llvm/test/tools/llvm-objcopy/wasm/only-keep-debug.test b/llvm/test/tools/llvm-objcopy/wasm/only-keep-debug.test
index fc57329bdd74c2f..c7a5da99b1edf70 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/only-keep-debug.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/only-keep-debug.test
@@ -1,26 +1,32 @@
 ## Test that only debug sections are kept with --only-keep-debug.
-# RUN: yaml2obj %s -o %t
-# RUN: llvm-strip --only-keep-debug %t
-# RUN: obj2yaml %t | FileCheck %s
+# RUN: yaml2obj %s -o %t.wasm
+# RUN: llvm-strip --only-keep-debug %t.wasm -o %t2.wasm
+# RUN: obj2yaml %t2.wasm | FileCheck %s -check-prefixes=CHECK,STRIPTYPE,STRIPFOO
 
 ## Test that keep-section overrides only-keep-debug.
-# RUN: yaml2obj %s -o %t
-# RUN: llvm-strip --only-keep-debug --keep-section=foo %t
-# RUN: obj2yaml %t | FileCheck --implicit-check-not=Name --check-prefix=CHECK --check-prefix=KEEP %s
+# RUN: llvm-strip --only-keep-debug --keep-section=foo %t.wasm -o %t3.wasm
+# RUN: obj2yaml %t3.wasm | FileCheck --implicit-check-not=Name --check-prefixes=CHECK,STRIPTYPE,KEEPFOO %s
 
 ## Test that keep-section overrides only-keep-debug, even for known sections.
-# RUN: yaml2obj %s -o %t
-# RUN: llvm-strip --only-keep-debug --keep-section=TYPE %t
-# RUN: obj2yaml %t | FileCheck --implicit-check-not=Name --check-prefix=CHECK --check-prefix=KEEPTYPE %s
+# RUN: llvm-strip --only-keep-debug --keep-section=TYPE %t.wasm -o %t4.wasm
+# RUN: obj2yaml %t4.wasm | FileCheck --implicit-check-not=Name --check-prefixes=CHECK,KEEPTYPE,STRIPFOO %s
 
 # CHECK:      Sections:
-# KEEPTYPE:     - Type: TYPE
-# CHECK-NOT:    - Type: TYPE
-# CHECK:        - Type: CUSTOM
-# CHECK-NEXT:     Name: .debug_info
-# CHECK:        - Type: CUSTOM
-# CHECK-NEXT:     Name: .debug_line
-# KEEP:           Name: foo
+# KEEPTYPE:       - Type: TYPE
+# STRIPTYPE-NOT:  - Type: TYPE
+# STRIPTYPE:      - Type: CUSTOM
+# STRIPTYPE-NEXT:   Name: .objcopy.removed
+# CHECK:          - Type: CUSTOM
+# CHECK-NEXT:       Name: .debug_info
+# CHECK:          - Type: CUSTOM
+# CHECK-NEXT:       Name: .objcopy.removed
+# CHECK:          - Type: CUSTOM
+# CHECK-NEXT:       Name: .objcopy.removed
+# CHECK:          - Type: CUSTOM
+# CHECK-NEXT:       Name: .debug_line
+# CHECK:          - Type: CUSTOM
+# KEEPFOO-NEXT:     Name: foo
+# STRIPFOO-NEXT:    Name: .objcopy.removed
 
 ## Test that remove-section overrides only-keep-debug.
 # RUN: yaml2obj %s -o %t
diff --git a/llvm/test/tools/llvm-objcopy/wasm/strip-all.test b/llvm/test/tools/llvm-objcopy/wasm/strip-all.test
index 9a9015713242c81..ef7c1a90e610752 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/strip-all.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/strip-all.test
@@ -7,6 +7,12 @@
 # CHECK:      Sections:
 # CHECK-NEXT:   - Type: TYPE
 # CHECK:        - Type: CUSTOM
+# CHECK-NEXT:        Name: .objcopy.removed
+# CHECK:        - Type: CUSTOM
+# CHECK-NEXT:        Name: .objcopy.removed
+# CHECK:        - Type: CUSTOM
+# CHECK-NEXT:        Name: .objcopy.removed
+# CHECK:        - Type: CUSTOM
 # CHECK-NEXT:        Name: foo
 
 --- !WASM
diff --git a/llvm/test/tools/llvm-objcopy/wasm/strip-debug.test b/llvm/test/tools/llvm-objcopy/wasm/strip-debug.test
index 2747c3bab742ea4..53e8153dd3539ab 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/strip-debug.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/strip-debug.test
@@ -1,14 +1,65 @@
-## Test that debug sections (but not linking or names) are stripped with --strip-debug
 # RUN: yaml2obj %s -o %t
-# RUN: llvm-strip --strip-debug %t
-# RUN: obj2yaml %t | FileCheck --implicit-check-not=.debug %s
+# RUN: cp %t %t3
+# RUN: llvm-objcopy --strip-debug %t %t2
+## Test that debug sections (but not linking or names) are stripped with --strip-debug
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not=.debug %s
+#
+# RUN: llvm-objcopy -g %t %t2g
+# Verify that --strip-debug and -g produce the same output
+# RUN: cmp %t2 %t2g
+
+# RUN: llvm-strip --strip-debug %t3
+# RUN: cmp %t2 %t3
+
+# RUN: cp %t %t4
+# RUN: llvm-strip -d %t4
+# RUN: cmp %t2 %t4
+
+# RUN: cp %t %t5
+# RUN: llvm-strip -g %t5
+# RUN: cmp %t2 %t5
+
+# RUN: cp %t %t6
+# RUN: llvm-strip -S %t6
+# RUN: cmp %t2 %t6
+
+# Verify that an archive with multiple object files is handled correctly.
+# RUN: cp %t %t.duplicate
+# RUN: cp %t2 %t.duplicate.stripped
+# RUN: rm -f %t.multiple-stripped-obj.a
+# RUN: llvm-ar crs %t.multiple-stripped-obj.a %t2 %t.duplicate.stripped
+# RUN: rm -f %t.multiple-obj.a
+# RUN: llvm-ar crs %t.multiple-obj.a %t %t.duplicate
+# RUN: llvm-objcopy --strip-debug %t.multiple-obj.a %t.multiple-obj.stripped.a
+# RUN: llvm-ar p %t.multiple-stripped-obj.a > %t.multiple-stripped-obj.a.dump
+# RUN: llvm-ar p %t.multiple-obj.stripped.a > %t.multiple-obj.stripped.a.dump
+# RUN: cmp %t.multiple-stripped-obj.a.dump %t.multiple-obj.stripped.a.dump
 
 # CHECK:      Sections:
 # CHECK-NEXT:   - Type: TYPE
-# CHECK:          Name: linking
+# CHECK:        - Type: CUSTOM
+## We expect the linking section to be preceeded by the removed `.debug_info`
+## section.
+# CHECK-NEXT:     Name:            .objcopy.removed
+# CHECK-NEXT:     Payload:         ''
+# CHECK-NEXT:   - Type: CUSTOM
+# CHECK-NEXT:     Name: linking
 # CHECK:          Name: name
 # CHECK-NEXT:     FunctionNames:
 # CHECK:          Name: producers
+## Following the producers section we expect to find three removed sections.
+## The `.debug_line` section that two reloction section corresponding to the
+## two debug sections.
+# CHECK:        - Type:            CUSTOM
+# CHECK-NEXT:     Name:            .objcopy.removed
+# CHECK-NEXT:     Payload:         ''
+# CHECK-NEXT:   - Type:            CUSTOM
+# CHECK-NEXT:     Name:            .objcopy.removed
+# CHECK-NEXT:     Payload:         ''
+# CHECK-NEXT:   - Type:            CUSTOM
+# CHECK-NEXT:     Name:            .objcopy.removed
+# CHECK-NEXT:     Payload:         ''
+
 
 --- !WASM
 FileHeader:
@@ -28,7 +79,11 @@ Sections:
         Body: 0B
   - Type: CUSTOM
     Name: .debug_info
-    Payload: CAFE1234
+    Payload: 'CAFE123456'
+    Relocations:
+      - Type:   R_WASM_FUNCTION_INDEX_LEB
+        Index:  0
+        Offset: 0x0000000
   - Type:            CUSTOM
     Name:            linking
     Version:         2
@@ -50,4 +105,8 @@ Sections:
         Version: 9.0.0
   - Type: CUSTOM
     Name: .debug_line
-    Payload: DEADBEEF
+    Payload: 'DEADBEEF01'
+    Relocations:
+      - Type:   R_WASM_FUNCTION_INDEX_LEB
+        Index:  0
+        Offset: 0x0000000
diff --git a/llvm/test/tools/llvm-objcopy/wasm/strip-reloc.test b/llvm/test/tools/llvm-objcopy/wasm/strip-reloc.test
index 4c8d1ba670447f8..d072bf70c9db76b 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/strip-reloc.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/strip-reloc.test
@@ -5,13 +5,15 @@
 
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy --strip-all %t %t2
-# RUN: obj2yaml %t2 | FileCheck --implicit-check-not=Type: %s
+# RUN: obj2yaml %t2 | FileCheck %s
 
 ## Check that the known sections are still present.
 # CHECK: Sections:
 # CHECK: - Type: TYPE
 # CHECK: - Type: FUNCTION
 # CHECK: - Type: CODE
+# CHECK-NOT: Relocations
+# CHECK-NOT: linking
 ## Check that there are still functions in the code section.
 # CHECK: Functions:
 
diff --git a/llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/print-mrs.obj.macho-aarch64 b/llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/print-mrs.obj.macho-aarch64
deleted file mode 100644
index 06cb13d07dafc5f..000000000000000
Binary files a/llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/print-mrs.obj.macho-aarch64 and /dev/null differ
diff --git a/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-print-mrs.test b/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-print-mrs.test
deleted file mode 100644
index cd40f0e849c5e8b..000000000000000
--- a/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-print-mrs.test
+++ /dev/null
@@ -1,3 +0,0 @@
-RUN: llvm-objdump -d -m --no-show-raw-insn %p/Inputs/print-mrs.obj.macho-aarch64 | FileCheck %s
-
-CHECK: 0:  mrs x0, CPM_IOACC_CTL_EL3
diff --git a/llvm/test/tools/llvm-objdump/XCOFF/private-headers.test b/llvm/test/tools/llvm-objdump/XCOFF/private-headers.test
new file mode 100644
index 000000000000000..9f1e60705bedf3c
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/XCOFF/private-headers.test
@@ -0,0 +1,52 @@
+## Test the --private-headers option for XCOFF object files.
+
+# RUN: yaml2obj -DMAGIC=0x1DF --docnum=1 %s -o %t_xcoff32.o
+# RUN: yaml2obj -DMAGIC=0x1F7 --docnum=1 %s -o %t_xcoff64.o
+# RUN: llvm-objdump --private-headers %t_xcoff32.o | \
+# RUN:   FileCheck %s --check-prefixes=CHECK32 --match-full-lines --strict-whitespace
+# RUN: llvm-objdump --private-headers %t_xcoff64.o | \
+# RUN:   FileCheck %s --check-prefixes=CHECK64 --match-full-lines --strict-whitespace
+
+--- !XCOFF
+FileHeader:
+  MagicNumber:       [[MAGIC]]
+  CreationTime:      1234
+Sections:
+  - Name:            .text
+    Flags:           [ STYP_TEXT ]
+    SectionData:     "9061FFF880820000"
+  - Name:            .data
+    Flags:           [ STYP_DATA ]
+    SectionData:     "0000000000000FC0"
+
+#      CHECK32:---File Header:
+# CHECK32-NEXT:Magic:              0x1df
+# CHECK32-NEXT:NumberOfSections:   2
+# CHECK32-NEXT:Timestamp:          1970-01-01 00:20:34 (1234)
+# CHECK32-NEXT:SymbolTableOffset:  0x0
+# CHECK32-NEXT:SymbolTableEntries: 0
+# CHECK32-NEXT:OptionalHeaderSize: 0x0
+# CHECK32-NEXT:Flags:              0x0
+
+#      CHECK64:---File Header:
+# CHECK64-NEXT:Magic:              0x1f7
+# CHECK64-NEXT:NumberOfSections:   2
+# CHECK64-NEXT:Timestamp:          1970-01-01 00:20:34 (1234)
+# CHECK64-NEXT:SymbolTableOffset:  0x0
+# CHECK64-NEXT:SymbolTableEntries: 0
+# CHECK64-NEXT:OptionalHeaderSize: 0x0
+# CHECK64-NEXT:Flags:              0x0
+
+## Test if the creation time of XCOFF is zero and the number of symbols is negative.
+# RUN: yaml2obj -DMAGIC=0x1DF --docnum=2 %s -o %t_xcoff_timestamp.o
+# RUN: llvm-objdump --private-headers %t_xcoff_timestamp.o | \
+# RUN:   FileCheck %s --match-full-lines
+
+--- !XCOFF
+FileHeader:
+  MagicNumber:          0x1DF
+  CreationTime:         0
+  EntriesInSymbolTable: -1
+
+# CHECK: Timestamp:          None (0)
+# CHECK: SymbolTableEntries: Reserved Value (-1)
diff --git a/llvm/test/tools/llvm-objdump/macho-stabs-in-syms.yaml b/llvm/test/tools/llvm-objdump/macho-stabs-in-syms.yaml
new file mode 100644
index 000000000000000..3850473b219cead
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/macho-stabs-in-syms.yaml
@@ -0,0 +1,263 @@
+# Check we don't error out on MachO files with stabs symbol entries that
+# repurpose the section field so it's out of range for normal symbols.
+
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objdump --syms %t | FileCheck %s
+
+# CHECK: 0000000000000000      d  *UND*
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x100000C
+  cpusubtype:      0x0
+  filetype:        0x2
+  ncmds:           16
+  sizeofcmds:      744
+  flags:           0x200085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          4294967296
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __TEXT
+    vmaddr:          4294967296
+    vmsize:          16384
+    fileoff:         0
+    filesize:        16384
+    maxprot:         5
+    initprot:        5
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x100003FA0
+        size:            8
+        offset:          0x3FA0
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         00008052C0035FD6
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x100003FA8
+        size:            88
+        offset:          0x3FA8
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000001C000000000000001C000000000000001C00000002000000A03F00004000000040000000A83F00000000000040000000000000000000000000000000030000000C00010010000100000000000000000200000000
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          4294983680
+    vmsize:          16384
+    fileoff:         16384
+    filesize:        688
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_CHAINED_FIXUPS
+    cmdsize:         16
+    dataoff:         16384
+    datasize:        56
+  - cmd:             LC_DYLD_EXPORTS_TRIE
+    cmdsize:         16
+    dataoff:         16440
+    datasize:        48
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          16496
+    nsyms:           11
+    stroff:          16672
+    strsize:         120
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       9
+    iextdefsym:      9
+    nextdefsym:      2
+    iundefsym:       11
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         32
+    name:            12
+    Content:         '/usr/lib/dyld'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            73BD5072-2F45-39FD-8013-8102F002C240
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           917504
+    sdk:             918528
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         72942592
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        16288
+    stacksize:       0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 88171522
+      compatibility_version: 65536
+    Content:         '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         16488
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         16496
+    datasize:        0
+  - cmd:             LC_CODE_SIGNATURE
+    cmdsize:         16
+    dataoff:         16800
+    datasize:        272
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      18
+        Name:            _
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      9
+            Name:            _mh_execute_header
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      13
+            Name:            main
+            Flags:           0x0
+            Address:         0x3FA0
+            Other:           0x0
+            ImportName:      ''
+  NameList:
+    - n_strx:          1
+      n_type:          0x64
+      # This line has been changed from a real object file to move the section
+      # out of valid range for normal symbols.
+      n_sect:          42
+      n_desc:          0
+      n_value:         0
+    - n_strx:          28
+      n_type:          0x64
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+    - n_strx:          68
+      n_type:          0x64
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+    - n_strx:          74
+      n_type:          0x66
+      n_sect:          0
+      n_desc:          1
+      n_value:         1713522175
+    - n_strx:          1
+      n_type:          0x2E
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983584
+    - n_strx:          22
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983584
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         8
+    - n_strx:          1
+      n_type:          0x4E
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983584
+    - n_strx:          1
+      n_type:          0x64
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          2
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          16
+      n_value:         4294967296
+    - n_strx:          22
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983584
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _main
+    - '/Users/tim/llvm/llvm-project/build.rel/'
+    - tmp.c
+    - '/Users/tim/llvm/llvm-project/build.rel/tmp.o'
+    - ''
+  FunctionStarts:  [ 0x3FA0 ]
+  ChainedFixups:   [ 0x0, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x30, 0x0,
+                     0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+...
diff --git a/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s
index a71f8a231545f2c..aaf28ed4a1e79a1 100644
--- a/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s
+++ b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s
@@ -108,12 +108,12 @@ end:
 # RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-0.s -o gnu-0x10000002-0.o
 # RUN: llvm-readelf --notes gnu-0x10000002-0.o | \
 # RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x0 (!PointerAuthIntrinsics, !PointerAuthCalls, !PointerAuthReturns, !PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0x0 (!PointerAuthIntrinsics, !PointerAuthCalls, !PointerAuthReturns, !PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT, !PointerAuthIndirectGotos, !PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)" %s
 # RUN: llvm-readobj --notes gnu-0x10000002-0.o | \
 # RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x0 (!PointerAuthIntrinsics, !PointerAuthCalls, !PointerAuthReturns, !PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0x0 (!PointerAuthIntrinsics, !PointerAuthCalls, !PointerAuthReturns, !PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT, !PointerAuthIndirectGotos, !PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)" %s
 
-#--- gnu-0x10000002-341.s
+#--- gnu-0x10000002-1365.s
 .section ".note.gnu.property", "a"
   .long 4           // Name length is always 4 ("GNU")
   .long end - begin // Data length
@@ -125,19 +125,19 @@ begin:
   .long 0xc0000001  // Type: GNU_PROPERTY_AARCH64_FEATURE_PAUTH
   .long 16          // Data size
   .quad 0x10000002  // PAuth ABI platform
-  .quad 341          // PAuth ABI version
+  .quad 1365        // PAuth ABI version
   .p2align 3        // Align to 8 byte for 64 bit
 end:
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-341.s -o gnu-0x10000002-341.o
-# RUN: llvm-readelf --notes gnu-0x10000002-341.o | \
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-1365.s -o gnu-0x10000002-1365.o
+# RUN: llvm-readelf --notes gnu-0x10000002-1365.o | \
 # RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x155 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)" %s
-# RUN: llvm-readobj --notes gnu-0x10000002-341.o | \
+# RUN:   -DVERSION="0x555 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, !PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)" %s
+# RUN: llvm-readobj --notes gnu-0x10000002-1365.o | \
 # RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x155 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0x555 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, !PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)" %s
 
-#--- gnu-0x10000002-170.s
+#--- gnu-0x10000002-2730.s
 .section ".note.gnu.property", "a"
   .long 4           // Name length is always 4 ("GNU")
   .long end - begin // Data length
@@ -149,19 +149,19 @@ begin:
   .long 0xc0000001  // Type: GNU_PROPERTY_AARCH64_FEATURE_PAUTH
   .long 16          // Data size
   .quad 0x10000002  // PAuth ABI platform
-  .quad 170         // PAuth ABI version
+  .quad 2730        // PAuth ABI version
   .p2align 3        // Align to 8 byte for 64 bit
 end:
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-170.s -o gnu-0x10000002-170.o
-# RUN: llvm-readelf --notes gnu-0x10000002-170.o | \
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-2730.s -o gnu-0x10000002-2730.o
+# RUN: llvm-readelf --notes gnu-0x10000002-2730.o | \
 # RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0xaa (!PointerAuthIntrinsics, PointerAuthCalls, !PointerAuthReturns, PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT)" %s
-# RUN: llvm-readobj --notes gnu-0x10000002-170.o | \
+# RUN:   -DVERSION="0xaaa (!PointerAuthIntrinsics, PointerAuthCalls, !PointerAuthReturns, PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT, PointerAuthIndirectGotos, !PointerAuthTypeInfoVTPtrDiscrimination, PointerAuthFPtrTypeDiscrimination)" %s
+# RUN: llvm-readobj --notes gnu-0x10000002-2730.o | \
 # RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0xaa (!PointerAuthIntrinsics, PointerAuthCalls, !PointerAuthReturns, PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0xaaa (!PointerAuthIntrinsics, PointerAuthCalls, !PointerAuthReturns, PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT, PointerAuthIndirectGotos, !PointerAuthTypeInfoVTPtrDiscrimination, PointerAuthFPtrTypeDiscrimination)" %s
 
-#--- gnu-0x10000002-511.s
+#--- gnu-0x10000002-4095.s
 .section ".note.gnu.property", "a"
   .long 4           // Name length is always 4 ("GNU")
   .long end - begin // Data length
@@ -173,19 +173,19 @@ begin:
   .long 0xc0000001  // Type: GNU_PROPERTY_AARCH64_FEATURE_PAUTH
   .long 16          // Data size
   .quad 0x10000002  // PAuth ABI platform
-  .quad 511         // PAuth ABI version
+  .quad 4095        // PAuth ABI version
   .p2align 3        // Align to 8 byte for 64 bit
 end:
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-511.s -o gnu-0x10000002-511.o
-# RUN: llvm-readelf --notes gnu-0x10000002-511.o | \
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-4095.s -o gnu-0x10000002-4095.o
+# RUN: llvm-readelf --notes gnu-0x10000002-4095.o | \
 # RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x1ff (PointerAuthIntrinsics, PointerAuthCalls, PointerAuthReturns, PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)" %s
-# RUN: llvm-readobj --notes gnu-0x10000002-511.o | \
+# RUN:   -DVERSION="0xfff (PointerAuthIntrinsics, PointerAuthCalls, PointerAuthReturns, PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, PointerAuthFPtrTypeDiscrimination)" %s
+# RUN: llvm-readobj --notes gnu-0x10000002-4095.o | \
 # RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x1ff (PointerAuthIntrinsics, PointerAuthCalls, PointerAuthReturns, PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0xfff (PointerAuthIntrinsics, PointerAuthCalls, PointerAuthReturns, PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, PointerAuthFPtrTypeDiscrimination)" %s
 
-#--- gnu-0x10000002-512.s
+#--- gnu-0x10000002-4096.s
 .section ".note.gnu.property", "a"
   .long 4           // Name length is always 4 ("GNU")
   .long end - begin // Data length
@@ -197,15 +197,15 @@ begin:
   .long 0xc0000001  // Type: GNU_PROPERTY_AARCH64_FEATURE_PAUTH
   .long 16          // Data size
   .quad 0x10000002  // PAuth ABI platform
-  .quad 512         // PAuth ABI version
+  .quad 4096        // PAuth ABI version
   .p2align 3        // Align to 8 byte for 64 bit
 end:
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-512.s -o gnu-0x10000002-512.o
-# RUN: llvm-readelf --notes gnu-0x10000002-512.o | \
-# RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" -DVERSION="0x200 (unknown)" %s
-# RUN: llvm-readobj --notes gnu-0x10000002-512.o | \
-# RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" -DVERSION="0x200 (unknown)" %s
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-4096.s -o gnu-0x10000002-4096.o
+# RUN: llvm-readelf --notes gnu-0x10000002-4096.o | \
+# RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" -DVERSION="0x1000 (unknown)" %s
+# RUN: llvm-readobj --notes gnu-0x10000002-4096.o | \
+# RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" -DVERSION="0x1000 (unknown)" %s
 
 #--- gnu-short.s
 .section ".note.gnu.property", "a"
diff --git a/llvm/test/tools/llvm-readobj/ELF/Sparc/elf-headers.test b/llvm/test/tools/llvm-readobj/ELF/Sparc/elf-headers.test
new file mode 100644
index 000000000000000..c05cd76640ed3eb
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/Sparc/elf-headers.test
@@ -0,0 +1,78 @@
+## Test various combinations of ELF flag values for SPARC.
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS32" -DMACHINE_NAME="EM_SPARC" -DFLAG_NAME=""
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON -DFLAG_VALUE=0x0
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x0 -DGNU_FLAG_NAME=""
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS32" -DMACHINE_NAME="EM_SPARC32PLUS" -DFLAG_NAME="EF_SPARC_32PLUS"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0 \
+# RUN:   -DFLAG0_NAME=EF_SPARC_32PLUS -DFLAG_VALUE=0x100
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x100 -DGNU_FLAG_NAME=", V8+ ABI"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS32" -DMACHINE_NAME="EM_SPARC32PLUS" \
+# RUN:   -DFLAG_NAME="EF_SPARC_32PLUS, EF_SPARC_SUN_US1"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0,LLVM-FLAG1\
+# RUN:   -DFLAG_VALUE=0x300 \
+# RUN:   -DFLAG0_NAME=EF_SPARC_32PLUS -DFLAG1_NAME=EF_SPARC_SUN_US1
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x300 \
+# RUN:   -DGNU_FLAG_NAME=", V8+ ABI, Sun UltraSPARC I extensions"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS32" -DMACHINE_NAME="EM_SPARC32PLUS" \
+# RUN:   -DFLAG_NAME="EF_SPARC_32PLUS, EF_SPARC_SUN_US1, EF_SPARC_HAL_R1, EF_SPARC_SUN_US3"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0,LLVM-FLAG1,LLVM-FLAG2,LLVM-FLAG3 \
+# RUN:   -DFLAG_VALUE=0xF00 \
+# RUN:   -DFLAG0_NAME=EF_SPARC_32PLUS -DFLAG1_NAME=EF_SPARC_HAL_R1 \
+# RUN:   -DFLAG2_NAME=EF_SPARC_SUN_US1 -DFLAG3_NAME=EF_SPARC_SUN_US3
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0xF00 \
+# RUN:   -DGNU_FLAG_NAME=", V8+ ABI, Sun UltraSPARC I extensions, HAL/Fujitsu R1 extensions, Sun UltraSPARC III extensions"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS64" -DMACHINE_NAME="EM_SPARCV9" -DFLAG_NAME=""
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON -DFLAG_VALUE=0x0
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x0 -DGNU_FLAG_NAME=""
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS64" -DMACHINE_NAME="EM_SPARCV9" -DFLAG_NAME="EF_SPARC_SUN_US1"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0 \
+# RUN:   -DFLAG_VALUE=0x200 -DFLAG0_NAME=EF_SPARC_SUN_US1
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x200 -DGNU_FLAG_NAME=", Sun UltraSPARC I extensions"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS64" -DMACHINE_NAME="EM_SPARCV9" \
+# RUN:   -DFLAG_NAME="EF_SPARC_SUN_US1, EF_SPARCV9_RMO"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0,LLVM-FLAG1 \
+# RUN:   -DFLAG_VALUE=0x202 \
+# RUN:   -DFLAG0_NAME=EF_SPARCV9_RMO -DFLAG1_NAME=EF_SPARC_SUN_US1
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x202 \
+# RUN:   -DGNU_FLAG_NAME=", Sun UltraSPARC I extensions, Relaxed Memory Ordering"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS64" -DMACHINE_NAME="EM_SPARCV9" \
+# RUN:   -DFLAG_NAME="EF_SPARC_SUN_US1, EF_SPARC_HAL_R1, EF_SPARC_SUN_US3, EF_SPARCV9_PSO"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0,LLVM-FLAG1,LLVM-FLAG2,LLVM-FLAG3 \
+# RUN:   -DFLAG_VALUE=0xE01 \
+# RUN:   -DFLAG0_NAME=EF_SPARCV9_PSO -DFLAG1_NAME=EF_SPARC_HAL_R1 \
+# RUN:   -DFLAG2_NAME=EF_SPARC_SUN_US1 -DFLAG3_NAME=EF_SPARC_SUN_US3
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0xE01\
+# RUN:   -DGNU_FLAG_NAME=", Sun UltraSPARC I extensions, HAL/Fujitsu R1 extensions, Sun UltraSPARC III extensions, Partial Store Ordering"
+
+--- !ELF
+FileHeader:
+  Class:           [[CLASS_NAME]]
+  Data:            ELFDATA2MSB
+  Type:            ET_REL
+  Machine:         [[MACHINE_NAME]]
+  Flags:           [ [[FLAG_NAME]] ]
+
+# LLVM-COMMON:          Flags [ ([[FLAG_VALUE]])
+# LLVM-FLAG0-NEXT:       [[FLAG0_NAME]]
+# LLVM-FLAG1-NEXT:       [[FLAG1_NAME]]
+# LLVM-FLAG2-NEXT:       [[FLAG2_NAME]]
+# LLVM-FLAG3-NEXT:       [[FLAG3_NAME]]
+# LLVM-COMMON-NEXT:     ]
+
+# GNU: Flags: [[FLAG_VALUE]][[GNU_FLAG_NAME]]
diff --git a/llvm/test/tools/llvm-split/AMDGPU/declarations-debug.ll b/llvm/test/tools/llvm-split/AMDGPU/declarations-debug.ll
new file mode 100644
index 000000000000000..66b89659f5d4f98
--- /dev/null
+++ b/llvm/test/tools/llvm-split/AMDGPU/declarations-debug.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-split -o %t %s -j 2 -mtriple amdgcn-amd-amdhsa --debug
+
+; REQUIRES: asserts
+
+; CHECK: --Partitioning Starts--
+; CHECK: P0 has a total cost of 0 (0.00% of source module)
+; CHECK: P1 has a total cost of 0 (0.00% of source module)
+; CHECK: --Partitioning Done--
+
+declare void @A()
+
+declare void @B()
diff --git a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
index cff5019f8e6ee4b..47aeb0ad8fdef9b 100644
--- a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
+++ b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
@@ -34,7 +34,7 @@
 #
 # CHECK: << Total TLI yes SDK no:  18
 # CHECK: >> Total TLI no  SDK yes: 0
-# CHECK: == Total TLI yes SDK yes: 248
+# CHECK: == Total TLI yes SDK yes: 250
 #
 # WRONG_DETAIL: << TLI yes SDK no : '_ZdaPv' aka operator delete[](void*)
 # WRONG_DETAIL: >> TLI no  SDK yes: '_ZdaPvj' aka operator delete[](void*, unsigned int)
@@ -48,14 +48,14 @@
 # WRONG_DETAIL: << TLI yes SDK no : 'fminimum_numl'
 # WRONG_SUMMARY: << Total TLI yes SDK no:  19{{$}}
 # WRONG_SUMMARY: >> Total TLI no  SDK yes: 1{{$}}
-# WRONG_SUMMARY: == Total TLI yes SDK yes: 247
+# WRONG_SUMMARY: == Total TLI yes SDK yes: 249
 #
 ## The -COUNT suffix doesn't care if there are too many matches, so check
 ## the exact count first; the two directives should add up to that.
 ## Yes, this means additions to TLI will fail this test, but the argument
 ## to -COUNT can't be an expression.
-# AVAIL: TLI knows 499 symbols, 266 available
-# AVAIL-COUNT-266: {{^}} available
+# AVAIL: TLI knows 501 symbols, 268 available
+# AVAIL-COUNT-268: {{^}} available
 # AVAIL-NOT:       {{^}} available
 # UNAVAIL-COUNT-233: not available
 # UNAVAIL-NOT:       not available
@@ -174,6 +174,10 @@ DynamicSymbols:
     Type:            STT_FUNC
     Section:         .text
     Binding:         STB_GLOBAL
+  - Name:            __cxa_throw
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
   - Name:            __cxa_guard_release
     Type:            STT_FUNC
     Section:         .text
@@ -286,6 +290,10 @@ DynamicSymbols:
     Type:            STT_FUNC
     Section:         .text
     Binding:         STB_GLOBAL
+  - Name:            _ZSt9terminatev
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
   - Name:            atof
     Type:            STT_FUNC
     Section:         .text
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index d3f7c2b7f819b68..80c84a977c26c64 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -130,8 +130,9 @@ static cl::opt<std::string> SplitDwarfFile(
 static cl::opt<bool> NoVerify("disable-verify", cl::Hidden,
                               cl::desc("Do not verify input module"));
 
-static cl::opt<bool> DisableSimplifyLibCalls("disable-simplify-libcalls",
-                                             cl::desc("Disable simplify-libcalls"));
+static cl::opt<bool>
+    DisableSimplifyLibCalls("disable-simplify-libcalls",
+                            cl::desc("Disable simplify-libcalls"));
 
 static cl::opt<bool> ShowMCEncoding("show-mc-encoding", cl::Hidden,
                                     cl::desc("Show encoding in .s output"));
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index bff05b9ca4bebc9..01f2033c397fb57 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -28,6 +28,8 @@
 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
+#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
+#include "llvm/ExecutionEngine/Orc/LoadRelocatableObject.h"
 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
 #include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h"
 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
@@ -260,6 +262,10 @@ static cl::opt<bool> UseSharedMemory(
     cl::desc("Use shared memory to transfer generated code and data"),
     cl::init(false), cl::cat(JITLinkCategory));
 
+static cl::opt<std::string>
+    OverrideTriple("triple", cl::desc("Override target triple detection"),
+                   cl::init(""), cl::cat(JITLinkCategory));
+
 static ExitOnError ExitOnErr;
 
 static LLVM_ATTRIBUTE_USED void linkComponents() {
@@ -1101,7 +1107,8 @@ Session::Session(std::unique_ptr<ExecutorProcessControl> EPC, Error &Err)
   for (auto &HarnessFile : TestHarnesses) {
     HarnessFiles.insert(HarnessFile);
 
-    auto ObjBuffer = ExitOnErr(getFile(HarnessFile));
+    auto ObjBuffer =
+        ExitOnErr(loadRelocatableObject(HarnessFile, ES.getTargetTriple()));
 
     auto ObjInterface =
         ExitOnErr(getObjectFileInterface(ES, ObjBuffer->getMemBufferRef()));
@@ -1418,6 +1425,14 @@ Session::findSymbolInfo(StringRef SymbolName, Twine ErrorMsgStem) {
 static std::pair<Triple, SubtargetFeatures> getFirstFileTripleAndFeatures() {
   static std::pair<Triple, SubtargetFeatures> FirstTTAndFeatures = []() {
     assert(!InputFiles.empty() && "InputFiles can not be empty");
+
+    if (!OverrideTriple.empty()) {
+      LLVM_DEBUG({
+        dbgs() << "Triple from -triple override: " << OverrideTriple << "\n";
+      });
+      return std::make_pair(Triple(OverrideTriple), SubtargetFeatures());
+    }
+
     for (auto InputFile : InputFiles) {
       auto ObjBuffer = ExitOnErr(getFile(InputFile));
       file_magic Magic = identify_magic(ObjBuffer->getBuffer());
@@ -1436,13 +1451,25 @@ static std::pair<Triple, SubtargetFeatures> getFirstFileTripleAndFeatures() {
         SubtargetFeatures Features;
         if (auto ObjFeatures = Obj->getFeatures())
           Features = std::move(*ObjFeatures);
+
+        LLVM_DEBUG({
+          dbgs() << "Triple from " << InputFile << ": " << TT.str() << "\n";
+        });
         return std::make_pair(TT, Features);
       }
       default:
         break;
       }
     }
-    return std::make_pair(Triple(), SubtargetFeatures());
+
+    // If no plain object file inputs exist to pin down the triple then detect
+    // the host triple and default to that.
+    auto JTMB = ExitOnErr(JITTargetMachineBuilder::detectHost());
+    LLVM_DEBUG({
+      dbgs() << "Triple from host-detection: " << JTMB.getTargetTriple().str()
+             << "\n";
+    });
+    return std::make_pair(JTMB.getTargetTriple(), JTMB.getFeatures());
   }();
 
   return FirstTTAndFeatures;
@@ -1711,9 +1738,9 @@ static Error addSectCreates(Session &S,
                                          ", filename component cannot be empty",
                                      inconvertibleErrorCode());
 
-    auto Content = MemoryBuffer::getFile(FileName);
+    auto Content = getFile(FileName);
     if (!Content)
-      return createFileError(FileName, errorCodeToError(Content.getError()));
+      return Content.takeError();
 
     SectCreateMaterializationUnit::ExtraSymbolsMap ExtraSymbols;
     while (!ExtraSymbolsString.empty()) {
@@ -1745,7 +1772,7 @@ static Error addTestHarnesses(Session &S) {
   LLVM_DEBUG(dbgs() << "Adding test harness objects...\n");
   for (auto HarnessFile : TestHarnesses) {
     LLVM_DEBUG(dbgs() << "  " << HarnessFile << "\n");
-    auto ObjBuffer = getFile(HarnessFile);
+    auto ObjBuffer = loadRelocatableObject(HarnessFile, S.ES.getTargetTriple());
     if (!ObjBuffer)
       return ObjBuffer.takeError();
     if (auto Err = S.ObjLayer.add(*S.MainJD, std::move(*ObjBuffer)))
@@ -1770,7 +1797,7 @@ static Error addObjects(Session &S,
     auto &JD = *std::prev(IdxToJD.lower_bound(InputFileArgIdx))->second;
     LLVM_DEBUG(dbgs() << "  " << InputFileArgIdx << ": \"" << InputFile
                       << "\" to " << JD.getName() << "\n";);
-    auto ObjBuffer = getFile(InputFile);
+    auto ObjBuffer = loadRelocatableObject(InputFile, S.ES.getTargetTriple());
     if (!ObjBuffer)
       return ObjBuffer.takeError();
 
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index 2b2cd906219a938..898d79b9233b9a3 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -104,6 +104,10 @@ static cl::opt<bool>
                      cl::desc("Preserve Comments in outputted assembly"),
                      cl::cat(MCCategory));
 
+static cl::opt<unsigned> CommentColumn("comment-column",
+                                       cl::desc("Asm comments indentation"),
+                                       cl::init(40));
+
 enum OutputFileType {
   OFT_Null,
   OFT_AssemblyFile,
@@ -405,6 +409,7 @@ int main(int argc, char **argv) {
     }
   }
   MAI->setPreserveAsmComments(PreserveComments);
+  MAI->setCommentColumn(CommentColumn);
 
   // Package up features to be passed to target/subtarget
   std::string FeaturesStr;
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
index 409a7010b80c8d6..ece5b6709a2e4c7 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
@@ -641,5 +641,51 @@ void BottleneckAnalysis::printView(raw_ostream &OS) const {
   printCriticalSequence(OS);
 }
 
+json::Value BottleneckAnalysis::toJSON() const {
+  if (!SeenStallCycles || !BPI.PressureIncreaseCycles) {
+    json::Object JO({{"PressureIncreaseCycles", 0}});
+    return JO;
+  }
+
+  json::Array CriticalSequence;
+  // get critical sequence
+  SmallVector<const DependencyEdge *, 16> Seq;
+  DG.getCriticalSequence(Seq);
+  if (!Seq.empty()) {
+    for (const DependencyEdge *&DE : Seq) {
+      json::Object DEJO({{"FromID", DE->FromIID},
+                         {"ToID", DE->ToIID},
+                         {"Type", static_cast<unsigned>(DE->Dep.Type)},
+                         {"ResourceOrRegID", DE->Dep.ResourceOrRegID}});
+      CriticalSequence.push_back(std::move(DEJO));
+    }
+  }
+
+  json::Array ResourcePressure;
+  if (BPI.PressureIncreaseCycles) {
+    ArrayRef<unsigned> Distribution = Tracker.getResourcePressureDistribution();
+    const MCSchedModel &SM = getSubTargetInfo().getSchedModel();
+    for (unsigned I = 0, E = Distribution.size(); I < E; ++I) {
+      unsigned ReleaseAtCycles = Distribution[I];
+      if (ReleaseAtCycles) {
+        const MCProcResourceDesc &PRDesc = *SM.getProcResource(I);
+        json::Object RPJO({{PRDesc.Name, ReleaseAtCycles}});
+        ResourcePressure.push_back(std::move(RPJO));
+      }
+    }
+  }
+
+  json::Object JO({{"PressureIncreaseCycles", BPI.PressureIncreaseCycles},
+                   {"ResourcePressureCycles", BPI.ResourcePressureCycles},
+                   {"DataDependencyCycles", BPI.DataDependencyCycles},
+                   {"RegisterDependencyCycles", BPI.RegisterDependencyCycles},
+                   {"MemoryDependencyCycles", BPI.MemoryDependencyCycles},
+                   {"TotalCycles", TotalCycles},
+                   {"DependencyEdge", std::move(CriticalSequence)},
+                   {"ResourcePressure", std::move(ResourcePressure)}});
+
+  return JO;
+}
+
 } // namespace mca.
 } // namespace llvm
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
index e709b25c3f76803..529090cf543fc41 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
@@ -335,7 +335,8 @@ class BottleneckAnalysis : public InstructionView {
 
   void printView(raw_ostream &OS) const override;
   StringRef getNameAsString() const override { return "BottleneckAnalysis"; }
-  bool isSerializable() const override { return false; }
+  bool isSerializable() const override { return true; }
+  json::Value toJSON() const override;
 
 #ifndef NDEBUG
   void dump(raw_ostream &OS, MCInstPrinter &MCIP) const { DG.dump(OS, MCIP); }
diff --git a/llvm/tools/llvm-objdump/XCOFFDump.cpp b/llvm/tools/llvm-objdump/XCOFFDump.cpp
index d9c00c096209831..524bf07c372a9bc 100644
--- a/llvm/tools/llvm-objdump/XCOFFDump.cpp
+++ b/llvm/tools/llvm-objdump/XCOFFDump.cpp
@@ -20,7 +20,9 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Endian.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 
 using namespace llvm;
@@ -30,10 +32,87 @@ using namespace llvm::support;
 
 namespace {
 class XCOFFDumper : public objdump::Dumper {
+  const XCOFFObjectFile &Obj;
+  unsigned Width;
+
 public:
-  XCOFFDumper(const object::XCOFFObjectFile &O) : Dumper(O) {}
-  void printPrivateHeaders() override {}
+  XCOFFDumper(const object::XCOFFObjectFile &O) : Dumper(O), Obj(O) {}
+
+private:
+  void printPrivateHeaders() override;
+  void printFileHeader();
+  FormattedString formatName(StringRef Name);
+  void printHex(StringRef Name, uint64_t Value);
+  void printNumber(StringRef Name, uint64_t Value);
+  void printStrHex(StringRef Name, StringRef Str, uint64_t Value);
+  void setWidth(unsigned W) { Width = W; };
 };
+
+void XCOFFDumper::printPrivateHeaders() { printFileHeader(); }
+
+FormattedString XCOFFDumper::formatName(StringRef Name) {
+  return FormattedString(Name, Width, FormattedString::JustifyLeft);
+}
+
+void XCOFFDumper::printHex(StringRef Name, uint64_t Value) {
+  outs() << formatName(Name) << format_hex(Value, 0) << "\n";
+}
+
+void XCOFFDumper::printNumber(StringRef Name, uint64_t Value) {
+  outs() << formatName(Name) << format_decimal(Value, 0) << "\n";
+}
+
+void XCOFFDumper::printStrHex(StringRef Name, StringRef Str, uint64_t Value) {
+  outs() << formatName(Name) << Str << " (" << format_decimal(Value, 0)
+         << ")\n";
+}
+
+void XCOFFDumper::printFileHeader() {
+  setWidth(20);
+  outs() << "\n---File Header:\n";
+  printHex("Magic:", Obj.getMagic());
+  printNumber("NumberOfSections:", Obj.getNumberOfSections());
+
+  int32_t Timestamp = Obj.getTimeStamp();
+  if (Timestamp > 0) {
+    // This handling of the timestamp assumes that the host system's time_t is
+    // compatible with AIX time_t. If a platform is not compatible, the lit
+    // tests will let us know.
+    time_t TimeDate = Timestamp;
+
+    char FormattedTime[20] = {};
+
+    size_t BytesFormatted = std::strftime(FormattedTime, sizeof(FormattedTime),
+                                          "%F %T", std::gmtime(&TimeDate));
+    assert(BytesFormatted && "The size of the buffer FormattedTime is less "
+                             "than the size of the date/time string.");
+    (void)BytesFormatted;
+    printStrHex("Timestamp:", FormattedTime, Timestamp);
+  } else {
+    // Negative timestamp values are reserved for future use.
+    printStrHex("Timestamp:", Timestamp == 0 ? "None" : "Reserved Value",
+                Timestamp);
+  }
+
+  // The number of symbol table entries is an unsigned value in 64-bit objects
+  // and a signed value (with negative values being 'reserved') in 32-bit
+  // objects.
+  if (Obj.is64Bit()) {
+    printHex("SymbolTableOffset:", Obj.getSymbolTableOffset64());
+    printNumber("SymbolTableEntries:", Obj.getNumberOfSymbolTableEntries64());
+  } else {
+    printHex("SymbolTableOffset:", Obj.getSymbolTableOffset32());
+    int32_t SymTabEntries = Obj.getRawNumberOfSymbolTableEntries32();
+    if (SymTabEntries >= 0)
+      printNumber("SymbolTableEntries:", SymTabEntries);
+    else
+      printStrHex("SymbolTableEntries:", "Reserved Value", SymTabEntries);
+  }
+
+  printHex("OptionalHeaderSize:", Obj.getOptionalHeaderSize());
+  printHex("Flags:", Obj.getFlags());
+}
+
 } // namespace
 
 std::unique_ptr<objdump::Dumper>
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 768a976cd920275..b69d14b4e7609aa 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -2885,16 +2885,6 @@ void Dumper::printSymbol(const SymbolRef &Symbol,
     reportUniqueWarning(AddrOrErr.takeError());
     return;
   }
-  uint64_t Address = *AddrOrErr;
-  section_iterator SecI = unwrapOrError(Symbol.getSection(), FileName);
-  if (SecI != O.section_end() && shouldAdjustVA(*SecI))
-    Address += AdjustVMA;
-  if ((Address < StartAddress) || (Address > StopAddress))
-    return;
-  SymbolRef::Type Type =
-      unwrapOrError(Symbol.getType(), FileName, ArchiveName, ArchitectureName);
-  uint32_t Flags =
-      unwrapOrError(Symbol.getFlags(), FileName, ArchiveName, ArchitectureName);
 
   // Don't ask a Mach-O STAB symbol for its section unless you know that
   // STAB symbol's section field refers to a valid section index. Otherwise
@@ -2913,6 +2903,16 @@ void Dumper::printSymbol(const SymbolRef &Symbol,
                                  : unwrapOrError(Symbol.getSection(), FileName,
                                                  ArchiveName, ArchitectureName);
 
+  uint64_t Address = *AddrOrErr;
+  if (Section != O.section_end() && shouldAdjustVA(*Section))
+    Address += AdjustVMA;
+  if ((Address < StartAddress) || (Address > StopAddress))
+    return;
+  SymbolRef::Type Type =
+      unwrapOrError(Symbol.getType(), FileName, ArchiveName, ArchitectureName);
+  uint32_t Flags =
+      unwrapOrError(Symbol.getFlags(), FileName, ArchiveName, ArchitectureName);
+
   StringRef Name;
   if (Type == SymbolRef::ST_Debug && Section != O.section_end()) {
     if (Expected<StringRef> NameOrErr = Section->getName())
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 2b5e2f4a9d347e3..97633c0fe5a37ea 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -1684,6 +1684,16 @@ const EnumEntry<unsigned> ElfHeaderRISCVFlags[] = {
   ENUM_ENT(EF_RISCV_TSO, "TSO"),
 };
 
+const EnumEntry<unsigned> ElfHeaderSPARCFlags[] = {
+    ENUM_ENT(EF_SPARC_32PLUS, "V8+ ABI"),
+    ENUM_ENT(EF_SPARC_SUN_US1, "Sun UltraSPARC I extensions"),
+    ENUM_ENT(EF_SPARC_HAL_R1, "HAL/Fujitsu R1 extensions"),
+    ENUM_ENT(EF_SPARC_SUN_US3, "Sun UltraSPARC III extensions"),
+    ENUM_ENT(EF_SPARCV9_TSO, "Total Store Ordering"),
+    ENUM_ENT(EF_SPARCV9_PSO, "Partial Store Ordering"),
+    ENUM_ENT(EF_SPARCV9_RMO, "Relaxed Memory Ordering"),
+};
+
 const EnumEntry<unsigned> ElfHeaderAVRFlags[] = {
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AVR_ARCH_AVR1),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AVR_ARCH_AVR2),
@@ -3625,6 +3635,9 @@ template <class ELFT> void GNUELFDumper<ELFT>::printFileHeaders() {
         unsigned(ELF::EF_MIPS_ABI), unsigned(ELF::EF_MIPS_MACH));
   else if (e.e_machine == EM_RISCV)
     ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderRISCVFlags));
+  else if (e.e_machine == EM_SPARC32PLUS || e.e_machine == EM_SPARCV9)
+    ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderSPARCFlags),
+                          unsigned(ELF::EF_SPARCV9_MM));
   else if (e.e_machine == EM_AVR)
     ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderAVRFlags),
                           unsigned(ELF::EF_AVR_ARCH_MASK));
@@ -5287,8 +5300,13 @@ static bool printAArch64PAuthABICoreInfo(raw_ostream &OS, uint32_t DataSize,
     Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINIADDRDISC] =
         "InitFiniAddressDiscrimination";
     Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT] = "ELFGOT";
+    Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOTOS] = "IndirectGotos";
+    Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_TYPEINFOVPTRDISCR] =
+        "TypeInfoVTPtrDiscrimination";
+    Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR] =
+        "FPtrTypeDiscrimination";
 
-    static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT ==
+    static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR ==
                       AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST,
                   "Update when new enum items are defined");
 
@@ -7118,6 +7136,9 @@ template <class ELFT> void LLVMELFDumper<ELFT>::printFileHeaders() {
       }
     } else if (E.e_machine == EM_RISCV)
       W.printFlags("Flags", E.e_flags, ArrayRef(ElfHeaderRISCVFlags));
+    else if (E.e_machine == EM_SPARC32PLUS || E.e_machine == EM_SPARCV9)
+      W.printFlags("Flags", E.e_flags, ArrayRef(ElfHeaderSPARCFlags),
+                   unsigned(ELF::EF_SPARCV9_MM));
     else if (E.e_machine == EM_AVR)
       W.printFlags("Flags", E.e_flags, ArrayRef(ElfHeaderAVRFlags),
                    unsigned(ELF::EF_AVR_ARCH_MASK));
diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt
index f48d840a105952f..745e4d9fb74a4a6 100644
--- a/llvm/unittests/ADT/CMakeLists.txt
+++ b/llvm/unittests/ADT/CMakeLists.txt
@@ -63,6 +63,7 @@ add_llvm_unittest(ADTTests
   PostOrderIteratorTest.cpp
   PriorityWorklistTest.cpp
   RangeAdapterTest.cpp
+  RewriteBufferTest.cpp
   SCCIteratorTest.cpp
   STLExtrasTest.cpp
   STLForwardCompatTest.cpp
diff --git a/llvm/unittests/ADT/CountCopyAndMove.cpp b/llvm/unittests/ADT/CountCopyAndMove.cpp
index fe1e2f4a5b89fd5..022dbee40dc6f1c 100644
--- a/llvm/unittests/ADT/CountCopyAndMove.cpp
+++ b/llvm/unittests/ADT/CountCopyAndMove.cpp
@@ -10,6 +10,8 @@
 
 using namespace llvm;
 
+int CountCopyAndMove::DefaultConstructions = 0;
+int CountCopyAndMove::ValueConstructions = 0;
 int CountCopyAndMove::CopyConstructions = 0;
 int CountCopyAndMove::CopyAssignments = 0;
 int CountCopyAndMove::MoveConstructions = 0;
diff --git a/llvm/unittests/ADT/CountCopyAndMove.h b/llvm/unittests/ADT/CountCopyAndMove.h
index 126054427b81a35..c91d34bd8ebb507 100644
--- a/llvm/unittests/ADT/CountCopyAndMove.h
+++ b/llvm/unittests/ADT/CountCopyAndMove.h
@@ -12,6 +12,8 @@
 namespace llvm {
 
 struct CountCopyAndMove {
+  static int DefaultConstructions;
+  static int ValueConstructions;
   static int CopyConstructions;
   static int CopyAssignments;
   static int MoveConstructions;
@@ -19,8 +21,8 @@ struct CountCopyAndMove {
   static int Destructions;
   int val;
 
-  CountCopyAndMove() = default;
-  explicit CountCopyAndMove(int val) : val(val) {}
+  CountCopyAndMove() { ++DefaultConstructions; }
+  explicit CountCopyAndMove(int val) : val(val) { ++ValueConstructions; }
   CountCopyAndMove(const CountCopyAndMove &other) : val(other.val) {
     ++CopyConstructions;
   }
@@ -40,6 +42,8 @@ struct CountCopyAndMove {
   ~CountCopyAndMove() { ++Destructions; }
 
   static void ResetCounts() {
+    DefaultConstructions = 0;
+    ValueConstructions = 0;
     CopyConstructions = 0;
     CopyAssignments = 0;
     MoveConstructions = 0;
@@ -47,6 +51,11 @@ struct CountCopyAndMove {
     Destructions = 0;
   }
 
+  static int TotalConstructions() {
+    return DefaultConstructions + ValueConstructions + MoveConstructions +
+           CopyConstructions;
+  }
+
   static int TotalCopies() { return CopyConstructions + CopyAssignments; }
 
   static int TotalMoves() { return MoveConstructions + MoveAssignments; }
diff --git a/llvm/unittests/ADT/FunctionExtrasTest.cpp b/llvm/unittests/ADT/FunctionExtrasTest.cpp
index fc856a976946bf6..9809a92daac7246 100644
--- a/llvm/unittests/ADT/FunctionExtrasTest.cpp
+++ b/llvm/unittests/ADT/FunctionExtrasTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/FunctionExtras.h"
+#include "CountCopyAndMove.h"
 #include "gtest/gtest.h"
 
 #include <memory>
@@ -310,4 +311,37 @@ class Incomplete {};
 Incomplete incompleteFunction() { return {}; }
 const Incomplete incompleteFunctionConst() { return {}; }
 
+// Check that we can store a pointer-sized payload inline in the unique_function.
+TEST(UniqueFunctionTest, InlineStorageWorks) {
+  // We do assume a couple of implementation details of the unique_function here:
+  //  - It can store certain small-enough payload inline
+  //  - Inline storage size is at least >= sizeof(void*)
+  void *ptr = nullptr;
+  unique_function<void(void *)> UniqueFunctionWithInlineStorage{
+      [ptr](void *self) {
+        auto mid = reinterpret_cast<uintptr_t>(&ptr);
+        auto beg = reinterpret_cast<uintptr_t>(self);
+        auto end = reinterpret_cast<uintptr_t>(self) +
+                   sizeof(unique_function<void(void *)>);
+        // Make sure the address of the captured pointer lies somewhere within
+        // the unique_function object.
+        EXPECT_TRUE(mid >= beg && mid < end);
+      }};
+  UniqueFunctionWithInlineStorage(&UniqueFunctionWithInlineStorage);
+}
+
+// Check that the moved-from captured state is properly destroyed during
+// move construction/assignment.
+TEST(UniqueFunctionTest, MovedFromStateIsDestroyedCorrectly) {
+  CountCopyAndMove::ResetCounts();
+  {
+    unique_function<void()> CapturingFunction{
+        [Counter = CountCopyAndMove{}] {}};
+    unique_function<void()> CapturingFunctionMoved{
+        std::move(CapturingFunction)};
+  }
+  EXPECT_EQ(CountCopyAndMove::TotalConstructions(),
+            CountCopyAndMove::Destructions);
+}
+
 } // anonymous namespace
diff --git a/clang/unittests/Rewrite/RewriteBufferTest.cpp b/llvm/unittests/ADT/RewriteBufferTest.cpp
similarity index 96%
rename from clang/unittests/Rewrite/RewriteBufferTest.cpp
rename to llvm/unittests/ADT/RewriteBufferTest.cpp
index bf29c0ff928304e..3b367daa7806e7a 100644
--- a/clang/unittests/Rewrite/RewriteBufferTest.cpp
+++ b/llvm/unittests/ADT/RewriteBufferTest.cpp
@@ -6,11 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Core/RewriteBuffer.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
-using namespace clang;
 
 namespace {
 
@@ -32,7 +31,7 @@ static void tagRange(unsigned Offset, unsigned Len, StringRef tagName,
   raw_string_ostream(EndTag) << "</" << tagName << '>';
 
   Buf.InsertTextAfter(Offset, BeginTag);
-  Buf.InsertTextBefore(Offset+Len, EndTag);
+  Buf.InsertTextBefore(Offset + Len, EndTag);
 }
 
 TEST(RewriteBuffer, TagRanges) {
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index 3cba630867a83b3..d9eb81faac42add 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -25,6 +25,7 @@ set(ANALYSIS_TEST_SOURCES
   DDGTest.cpp
   DomTreeUpdaterTest.cpp
   DXILResourceTest.cpp
+  GraphWriterTest.cpp
   GlobalsModRefTest.cpp
   FunctionPropertiesAnalysisTest.cpp
   InlineCostTest.cpp
diff --git a/llvm/unittests/Analysis/GraphWriterTest.cpp b/llvm/unittests/Analysis/GraphWriterTest.cpp
new file mode 100644
index 000000000000000..a723c92d1576185
--- /dev/null
+++ b/llvm/unittests/Analysis/GraphWriterTest.cpp
@@ -0,0 +1,77 @@
+//===- GraphWriterTest.cpp - GraphWriter unit tests -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/Analysis/CFGPrinter.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+#include <string>
+
+#define ASSERT_NO_ERROR(x)                                                     \
+  if (std::error_code ASSERT_NO_ERROR_ec = x) {                                \
+    SmallString<128> MessageStorage;                                           \
+    raw_svector_ostream Message(MessageStorage);                               \
+    Message << #x ": did not return errc::success.\n"                          \
+            << "error number: " << ASSERT_NO_ERROR_ec.value() << "\n"          \
+            << "error message: " << ASSERT_NO_ERROR_ec.message() << "\n";      \
+    GTEST_FATAL_FAILURE_(MessageStorage.c_str());                              \
+  } else {                                                                     \
+  }
+
+namespace llvm {
+namespace {
+
+class GraphWriterTest : public testing::Test {
+protected:
+  LLVMContext C;
+
+  std::unique_ptr<Module> makeLLVMModule() {
+    const char *ModuleStrig = "define i32 @f(i32 %x) {\n"
+                              "bb0:\n"
+                              "  %y1 = icmp eq i32 %x, 0 \n"
+                              "  br i1 %y1, label %bb1, label %bb2 \n"
+                              "bb1:\n"
+                              "  br label %bb3\n"
+                              "bb2:\n"
+                              "  br label %bb3\n"
+                              "bb3:\n"
+                              "  %y2 = phi i32 [0, %bb1], [1, %bb2] \n"
+                              "  ret i32 %y2\n"
+                              "}\n";
+    SMDiagnostic Err;
+    return parseAssemblyString(ModuleStrig, Err, C);
+  }
+};
+
+static void writeCFGToDotFile(Function &F, std::string Name,
+                              bool CFGOnly = false) {
+  std::error_code EC;
+  raw_fd_ostream File(Name + ".dot", EC, sys::fs::OpenFlags::OF_Text);
+
+  DOTFuncInfo CFGInfo(&F);
+
+  ASSERT_NO_ERROR(EC);
+  // Test intentionally does not pass BPI, WriteGraph should work without it.
+  WriteGraph(File, &CFGInfo, CFGOnly);
+}
+
+TEST_F(GraphWriterTest, WriteCFGDotFileTest) {
+  auto M = makeLLVMModule();
+  Function *F = M->getFunction("f");
+
+  writeCFGToDotFile(*F, "test-full");
+  writeCFGToDotFile(*F, "test-cfg-only", true);
+}
+
+} // end anonymous namespace
+} // end namespace llvm
diff --git a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
index 1ba402276e036db..4c177ae8446905a 100644
--- a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
+++ b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Analysis/MemoryProfileInfo.h"
 #include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index ff7dec5bee31dfb..abaedf9c210b8c4 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -535,12 +535,14 @@ TEST_F(TargetLibraryInfoTest, ValidProto) {
       "declare void @__cxa_guard_abort(%struct*)\n"
       "declare i32 @__cxa_guard_acquire(%struct*)\n"
       "declare void @__cxa_guard_release(%struct*)\n"
+      "declare void @__cxa_throw(ptr, ptr, ptr)\n"
 
       "declare i32 @atexit(void ()*)\n"
 
       "declare void @abort()\n"
       "declare void @exit(i32)\n"
       "declare void @_Exit(i32)\n"
+      "declare void @_ZSt9terminatev()\n"
 
       "declare i32 @__nvvm_reflect(i8*)\n"
 
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index b9414be98623af2..e66584b81bba256 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -185,6 +185,7 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   SDValue Op0 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, Int32VT);
   SDValue Op1 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 2, Int32VT);
   SDValue Op2 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 3, Float32VT);
+  SDValue Op3 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 8, Int32VT);
 
   SDValue Add = DAG->getNode(ISD::ADD, DL, Int32VT, Op0, Op1);
   SDValue Sub = DAG->getNode(ISD::SUB, DL, Int32VT, Add, Op0);
@@ -192,6 +193,9 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   SDValue And = DAG->getNode(ISD::AND, DL, Int32VT, Op0, Op1);
   SDValue Xor = DAG->getNode(ISD::XOR, DL, Int32VT, Op1, Op0);
   SDValue Or  = DAG->getNode(ISD::OR, DL, Int32VT, Op0, Op1);
+  SDNodeFlags DisFlags;
+  DisFlags.setDisjoint(true);
+  SDValue DisOr = DAG->getNode(ISD::OR, DL, Int32VT, Op0, Op3, DisFlags);
   SDValue SMax = DAG->getNode(ISD::SMAX, DL, Int32VT, Op0, Op1);
   SDValue SMin = DAG->getNode(ISD::SMIN, DL, Int32VT, Op1, Op0);
   SDValue UMax = DAG->getNode(ISD::UMAX, DL, Int32VT, Op0, Op1);
@@ -205,6 +209,7 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   EXPECT_TRUE(sd_match(Sub, m_Sub(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(Add, m_c_BinOp(ISD::ADD, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(Add, m_Add(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(Add, m_AddLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(
       Mul, m_Mul(m_OneUse(m_Opc(ISD::SUB)), m_NUses<2>(m_Specific(Add)))));
   EXPECT_TRUE(
@@ -217,6 +222,12 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   EXPECT_TRUE(sd_match(Xor, m_Xor(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(Or, m_c_BinOp(ISD::OR, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(Or, m_Or(m_Value(), m_Value())));
+  EXPECT_FALSE(sd_match(Or, m_DisjointOr(m_Value(), m_Value())));
+
+  EXPECT_TRUE(sd_match(DisOr, m_Or(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(DisOr, m_DisjointOr(m_Value(), m_Value())));
+  EXPECT_FALSE(sd_match(DisOr, m_Add(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(DisOr, m_AddLike(m_Value(), m_Value())));
 
   EXPECT_TRUE(sd_match(SMax, m_c_BinOp(ISD::SMAX, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMax, m_SMax(m_Value(), m_Value())));
@@ -242,9 +253,14 @@ TEST_F(SelectionDAGPatternMatchTest, matchUnaryOp) {
 
   SDValue Op0 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, Int32VT);
   SDValue Op1 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, Int64VT);
-  SDValue Op2 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, FloatVT);
+  SDValue Op2 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, FloatVT);  
+  SDValue Op3 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 3, Int32VT);
 
   SDValue ZExt = DAG->getNode(ISD::ZERO_EXTEND, DL, Int64VT, Op0);
+  SDNodeFlags NNegFlags;
+  NNegFlags.setNonNeg(true);
+  SDValue ZExtNNeg =
+      DAG->getNode(ISD::ZERO_EXTEND, DL, Int64VT, Op3, NNegFlags);
   SDValue SExt = DAG->getNode(ISD::SIGN_EXTEND, DL, Int64VT, Op0);
   SDValue Trunc = DAG->getNode(ISD::TRUNCATE, DL, Int32VT, Op1);
 
@@ -260,6 +276,13 @@ TEST_F(SelectionDAGPatternMatchTest, matchUnaryOp) {
   using namespace SDPatternMatch;
   EXPECT_TRUE(sd_match(ZExt, m_UnaryOp(ISD::ZERO_EXTEND, m_Value())));
   EXPECT_TRUE(sd_match(SExt, m_SExt(m_Value())));
+  EXPECT_TRUE(sd_match(SExt, m_SExtLike(m_Value())));
+  ASSERT_TRUE(ZExtNNeg->getFlags().hasNonNeg());
+  EXPECT_FALSE(sd_match(ZExtNNeg, m_SExt(m_Value())));
+  EXPECT_TRUE(sd_match(ZExtNNeg, m_NNegZExt(m_Value())));
+  EXPECT_FALSE(sd_match(ZExt, m_NNegZExt(m_Value())));
+  EXPECT_TRUE(sd_match(ZExtNNeg, m_SExtLike(m_Value())));
+  EXPECT_FALSE(sd_match(ZExt, m_SExtLike(m_Value())));
   EXPECT_TRUE(sd_match(Trunc, m_Trunc(m_Specific(Op1))));
 
   EXPECT_TRUE(sd_match(Neg, m_Neg(m_Value())));
@@ -393,6 +416,10 @@ struct VPMatchContext : public SDPatternMatch::BasicMatchContext {
     auto BaseOpc = ISD::getBaseOpcodeForVP(OpVal->getOpcode(), false);
     return BaseOpc.has_value() && *BaseOpc == Opc;
   }
+
+  unsigned getNumOperands(SDValue N) const {
+    return N->isVPOpcode() ? N->getNumOperands() - 2 : N->getNumOperands();
+  }
 };
 } // anonymous namespace
 TEST_F(SelectionDAGPatternMatchTest, matchContext) {
@@ -410,15 +437,54 @@ TEST_F(SelectionDAGPatternMatchTest, matchContext) {
                                {Vector0, Vector0, Mask0, Scalar0});
   SDValue VPReduceAdd = DAG->getNode(ISD::VP_REDUCE_ADD, DL, Int32VT,
                                      {Scalar0, VPAdd, Mask0, Scalar0});
+  SDValue Add = DAG->getNode(ISD::ADD, DL, VInt32VT, {Vector0, Vector0});
 
   using namespace SDPatternMatch;
   VPMatchContext VPCtx(DAG.get());
   EXPECT_TRUE(sd_context_match(VPAdd, VPCtx, m_Opc(ISD::ADD)));
+  EXPECT_TRUE(
+      sd_context_match(VPAdd, VPCtx, m_Node(ISD::ADD, m_Value(), m_Value())));
+  // VPMatchContext can't match pattern using explicit VP Opcode
+  EXPECT_FALSE(sd_context_match(VPAdd, VPCtx,
+                                m_Node(ISD::VP_ADD, m_Value(), m_Value())));
+  EXPECT_FALSE(sd_context_match(
+      VPAdd, VPCtx,
+      m_Node(ISD::VP_ADD, m_Value(), m_Value(), m_Value(), m_Value())));
+  // Check Binary Op Pattern
+  EXPECT_TRUE(sd_context_match(VPAdd, VPCtx, m_Add(m_Value(), m_Value())));
   // VP_REDUCE_ADD doesn't have a based opcode, so we use a normal
   // sd_match before switching to VPMatchContext when checking VPAdd.
   EXPECT_TRUE(sd_match(VPReduceAdd, m_Node(ISD::VP_REDUCE_ADD, m_Value(),
                                            m_Context(VPCtx, m_Opc(ISD::ADD)),
                                            m_Value(), m_Value())));
+  // non-vector predicated should match too
+  EXPECT_TRUE(sd_context_match(Add, VPCtx, m_Opc(ISD::ADD)));
+  EXPECT_TRUE(
+      sd_context_match(Add, VPCtx, m_Node(ISD::ADD, m_Value(), m_Value())));
+  EXPECT_FALSE(sd_context_match(
+      Add, VPCtx,
+      m_Node(ISD::ADD, m_Value(), m_Value(), m_Value(), m_Value())));
+  EXPECT_TRUE(sd_context_match(Add, VPCtx, m_Add(m_Value(), m_Value())));
+}
+
+TEST_F(SelectionDAGPatternMatchTest, matchVPWithBasicContext) {
+  SDLoc DL;
+  auto BoolVT = EVT::getIntegerVT(Context, 1);
+  auto Int32VT = EVT::getIntegerVT(Context, 32);
+  auto VInt32VT = EVT::getVectorVT(Context, Int32VT, 4);
+  auto MaskVT = EVT::getVectorVT(Context, BoolVT, 4);
+
+  SDValue Vector0 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, VInt32VT);
+  SDValue Mask = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 2, MaskVT);
+  SDValue EL = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 3, Int32VT);
+
+  SDValue VPAdd =
+      DAG->getNode(ISD::VP_ADD, DL, VInt32VT, Vector0, Vector0, Mask, EL);
+
+  using namespace SDPatternMatch;
+  EXPECT_FALSE(sd_match(VPAdd, m_Node(ISD::VP_ADD, m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(
+      VPAdd, m_Node(ISD::VP_ADD, m_Value(), m_Value(), m_Value(), m_Value())));
 }
 
 TEST_F(SelectionDAGPatternMatchTest, matchAdvancedProperties) {
diff --git a/llvm/unittests/IR/DataLayoutTest.cpp b/llvm/unittests/IR/DataLayoutTest.cpp
index dcb2e614f4c40de..9464b489b65f43c 100644
--- a/llvm/unittests/IR/DataLayoutTest.cpp
+++ b/llvm/unittests/IR/DataLayoutTest.cpp
@@ -19,6 +19,8 @@ using namespace llvm;
 
 namespace {
 
+class DataLayoutTest : public ::testing::Test {};
+
 // TODO: Split into multiple TESTs.
 TEST(DataLayoutTest, ParseErrors) {
   EXPECT_THAT_EXPECTED(
@@ -30,26 +32,6 @@ TEST(DataLayoutTest, ParseErrors) {
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("n0"),
       FailedWithMessage("Zero width native integer type in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p16777216:64:64:64"),
-      FailedWithMessage("Invalid address space, must be a 24-bit integer"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("a1:64"),
-      FailedWithMessage("Sized aggregate specification in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("a:"),
-      FailedWithMessage("Trailing separator in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:48:52"),
-      FailedWithMessage("number of bits must be a byte width multiple"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("e-p"),
-      FailedWithMessage(
-          "Missing size specification for pointer in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("e-p:64"),
-      FailedWithMessage(
-          "Missing alignment specification for pointer in datalayout string"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("m"),
       FailedWithMessage("Expected mangling specifier in datalayout string"));
@@ -57,53 +39,10 @@ TEST(DataLayoutTest, ParseErrors) {
       DataLayout::parse("m."),
       FailedWithMessage("Unexpected trailing characters after mangling "
                         "specifier in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("f"),
-      FailedWithMessage(
-          "Missing alignment specification in datalayout string"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse(":32"),
       FailedWithMessage(
           "Expected token before separator in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i64:64:16"),
-      FailedWithMessage(
-          "Preferred alignment cannot be less than the ABI alignment"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i64:16:16777216"),
-      FailedWithMessage(
-          "Invalid preferred alignment, must be a 16bit integer"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i64:16777216:16777216"),
-      FailedWithMessage("Invalid ABI alignment, must be a 16bit integer"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i16777216:16:16"),
-      FailedWithMessage("Invalid bit width, must be a 24-bit integer"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:32:32:16"),
-      FailedWithMessage(
-          "Preferred alignment cannot be less than the ABI alignment"));
-  EXPECT_THAT_EXPECTED(DataLayout::parse("p:0:32:32"),
-                       FailedWithMessage("Invalid pointer size of 0 bytes"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:64:24:64"),
-      FailedWithMessage("Pointer ABI alignment must be a power of 2"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:64:64:24"),
-      FailedWithMessage("Pointer preferred alignment must be a power of 2"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:64:64:64:128"),
-      FailedWithMessage("Index width cannot be larger than pointer width"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("v128:0:128"),
-      FailedWithMessage(
-          "ABI alignment specification must be >0 for non-aggregate types"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i32:24:32"),
-      FailedWithMessage("Invalid ABI alignment, must be a power of 2"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i32:32:24"),
-      FailedWithMessage("Invalid preferred alignment, must be a power of 2"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("A16777216"),
       FailedWithMessage("Invalid address space, must be a 24-bit integer"));
@@ -116,14 +55,414 @@ TEST(DataLayoutTest, ParseErrors) {
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("Fi24"),
       FailedWithMessage("Alignment is neither 0 nor a power of 2"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i8:16"),
-      FailedWithMessage("Invalid ABI alignment, i8 must be naturally aligned"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("S24"),
       FailedWithMessage("Alignment is neither 0 nor a power of 2"));
 }
 
+TEST(DataLayout, LayoutStringFormat) {
+  for (StringRef Str : {"", "e", "m:e", "m:e-e"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
+
+  for (StringRef Str : {"-", "e-", "-m:e", "m:e--e"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("empty specification is not allowed"));
+}
+
+class DataLayoutPrimitiveSpecificationTest
+    : public DataLayoutTest,
+      public ::testing::WithParamInterface<char> {
+  char Specifier;
+
+public:
+  DataLayoutPrimitiveSpecificationTest() : Specifier(GetParam()) {}
+
+  std::string format(StringRef Str) const {
+    std::string Res = Str.str();
+    std::replace(Res.begin(), Res.end(), '!', Specifier);
+    return Res;
+  }
+};
+
+INSTANTIATE_TEST_SUITE_P(PrmitiveSpecifiers,
+                         DataLayoutPrimitiveSpecificationTest,
+                         ::testing::Values('i', 'f', 'v'));
+
+TEST_P(DataLayoutPrimitiveSpecificationTest, ParsePrimitiveSpec) {
+  for (StringRef Str :
+       {"!1:16", "!8:8:8", "!16:32:64", "!16777215:32768:32768"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(format(Str)), Succeeded());
+
+  for (StringRef Str : {"!", "!1", "!32:32:32:32", "!16:32:64:128"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage(format("malformed specification, must be of the form "
+                                 "\"!<size>:<abi>[:<pref>]\"")));
+
+  // size
+  for (StringRef Str : {"!:8", "!:16:16", "!:32:64"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(format(Str)),
+                         FailedWithMessage("size component cannot be empty"));
+
+  for (StringRef Str :
+       {"!0:8", "!0x8:8", "!x:8:8", "!0:16:32", "!16777216:64:64"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("size must be a non-zero 24-bit integer"));
+
+  // ABI alignment
+  for (StringRef Str : {"!8:", "!16::16", "!32::64"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("ABI alignment component cannot be empty"));
+
+  for (StringRef Str : {"!1:x", "!8:8x:8", "!16:65536:65536"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("ABI alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"!8:0", "!16:0:16", "!32:0:64"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(format(Str)),
+                         FailedWithMessage("ABI alignment must be non-zero"));
+
+  for (StringRef Str : {"!1:1", "!8:4", "!16:6:16", "!32:24:64"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage(
+            "ABI alignment must be a power of two times the byte width"));
+
+  // preferred alignment
+  for (StringRef Str : {"!1:8:", "!16:16:", "!64:32:"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("preferred alignment component cannot be empty"));
+
+  for (StringRef Str : {"!1:8:x", "!8:8:0x8", "!16:32:65536"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("preferred alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"!8:8:0", "!32:16:0"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("preferred alignment must be non-zero"));
+
+  for (StringRef Str : {"!1:8:12", "!8:8:17", "!16:32:40"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage(
+            "preferred alignment must be a power of two times the byte width"));
+
+  for (StringRef Str : {"!1:16:8", "!64:32:16"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage(
+            "preferred alignment cannot be less than the ABI alignment"));
+
+  // Additional check for byte-sized integer.
+  if (GetParam() == 'i') {
+    for (StringRef Str : {"!8:16", "!8:16:8", "!8:16:32"})
+      EXPECT_THAT_EXPECTED(DataLayout::parse(format(Str)),
+                           FailedWithMessage("i8 must be 8-bit aligned"));
+  }
+}
+
+TEST(DataLayoutTest, ParseAggregateSpec) {
+  for (StringRef Str : {"a:8", "a:0:16", "a0:32:64", "a:32768:32768"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
+
+  for (StringRef Str : {"a", "a0", "a:32:32:32", "a0:32:64:128"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("malformed specification, must be of the form "
+                          "\"a:<abi>[:<pref>]\""));
+
+  // size
+  for (StringRef Str : {"a1:8", "a0x0:8", "ax:16:32"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str),
+                         FailedWithMessage("size must be zero"));
+
+  // ABI alignment
+  for (StringRef Str : {"a:", "a0:", "a::32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("ABI alignment component cannot be empty"));
+
+  for (StringRef Str : {"a:x", "a0:0x0", "a:65536", "a0:65536:65536"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("ABI alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"a:1", "a:4", "a:9:16", "a0:24:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "ABI alignment must be a power of two times the byte width"));
+
+  // preferred alignment
+  for (StringRef Str : {"a:8:", "a0:16:", "a0:0:"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment component cannot be empty"));
+
+  for (StringRef Str : {"a:16:x", "a0:8:0x8", "a:16:65536"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"a:0:0", "a0:16:0"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment must be non-zero"));
+
+  for (StringRef Str : {"a:8:12", "a:16:17", "a0:32:40"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "preferred alignment must be a power of two times the byte width"));
+
+  for (StringRef Str : {"a:16:8", "a0:32:16"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "preferred alignment cannot be less than the ABI alignment"));
+}
+
+TEST(DataLayout, ParsePointerSpec) {
+  for (StringRef Str :
+       {"p:16:8", "p:16:16:64", "p:32:64:64:32", "p0:32:64", "p42:64:32:32",
+        "p16777215:32:32:64:8", "p16777215:16777215:32768:32768:16777215"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
+
+  for (StringRef Str :
+       {"p", "p0", "p:32", "p0:32", "p:32:32:32:32:32", "p0:32:32:32:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("malformed specification, must be of the form "
+                          "\"p[<n>]:<size>:<abi>[:<pref>[:<idx>]]\""));
+
+  // address space
+  for (StringRef Str : {"p0x0:32:32", "px:32:32:32", "p16777216:32:32:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("address space must be a 24-bit integer"));
+
+  // pointer size
+  for (StringRef Str : {"p::32", "p0::32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("pointer size component cannot be empty"));
+
+  for (StringRef Str : {"p:0:32", "p0:0x1:32:32", "p42:16777216:32:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("pointer size must be a non-zero 24-bit integer"));
+
+  // ABI alignment
+  for (StringRef Str : {"p:32:", "p0:32::32", "p42:32::32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("ABI alignment component cannot be empty"));
+
+  for (StringRef Str : {"p:32:x", "p0:32:0x20:32", "p42:32:65536:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("ABI alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"p:32:0", "p0:32:0:32", "p42:32:0:32:32"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str),
+                         FailedWithMessage("ABI alignment must be non-zero"));
+
+  for (StringRef Str : {"p:32:4", "p42:32:24:32", "p0:32:65535:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "ABI alignment must be a power of two times the byte width"));
+
+  // preferred alignment
+  for (StringRef Str : {"p:32:32:", "p0:32:32:", "p42:32:32::32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment component cannot be empty"));
+
+  for (StringRef Str : {"p:32:32:x", "p0:32:32:0x20", "p42:32:32:65536:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"p:32:32:0", "p0:32:32:0", "p42:32:32:0:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment must be non-zero"));
+
+  for (StringRef Str : {"p:32:32:4", "p0:32:32:24", "p42:32:32:65535:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "preferred alignment must be a power of two times the byte width"));
+
+  for (StringRef Str : {"p:64:64:32", "p0:16:32:16:16"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "preferred alignment cannot be less than the ABI alignment"));
+
+  // index size
+  for (StringRef Str : {"p:32:32:32:", "p0:32:32:32:"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("index size component cannot be empty"));
+
+  for (StringRef Str :
+       {"p:32:32:32:0", "p0:32:32:32:0x20", "p42:32:32:32:16777216"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("index size must be a non-zero 24-bit integer"));
+
+  for (StringRef Str : {"p:16:16:16:17", "p0:32:64:64:64", "p42:16:64:64:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("index size cannot be larger than the pointer size"));
+}
+
+TEST(DataLayout, ParseNonIntegralAddrSpace) {
+  for (StringRef Str : {"ni:1", "ni:16777215", "ni:1:16777215"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
+
+  for (StringRef Str : {"ni", "ni42", "nix"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("malformed specification, must be of the form "
+                          "\"ni:<address space>[:<address space>]...\""));
+
+  for (StringRef Str : {"ni:", "ni::42", "ni:42:"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("address space component cannot be empty"));
+
+  for (StringRef Str : {"ni:x", "ni:42:0x1", "ni:16777216", "ni:42:16777216"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("address space must be a 24-bit integer"));
+
+  for (StringRef Str : {"ni:0", "ni:42:0"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("address space 0 cannot be non-integral"));
+}
+
+TEST(DataLayout, GetPointerSizeInBits) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 64, 64, 64},
+      {"p:16:32", 16, 16, 16},
+      {"p0:32:64", 32, 32, 32},
+      {"p1:16:32", 64, 16, 64},
+      {"p1:31:32-p2:15:16:16:14", 64, 31, 15},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getPointerSizeInBits(0), V0) << Layout;
+    EXPECT_EQ(DL.getPointerSizeInBits(1), V1) << Layout;
+    EXPECT_EQ(DL.getPointerSizeInBits(2), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetPointerSize) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 8, 8, 8},
+      {"p:16:32", 2, 2, 2},
+      {"p0:32:64", 4, 4, 4},
+      {"p1:17:32", 8, 3, 8},
+      {"p1:31:64-p2:23:8:16:9", 8, 4, 3},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getPointerSize(0), V0) << Layout;
+    EXPECT_EQ(DL.getPointerSize(1), V1) << Layout;
+    EXPECT_EQ(DL.getPointerSize(2), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetIndexSizeInBits) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 64, 64, 64},
+      {"p:16:32", 16, 16, 16},
+      {"p0:32:64", 32, 32, 32},
+      {"p1:16:32:32:10", 64, 10, 64},
+      {"p1:31:32:64:20-p2:17:16:16:15", 64, 20, 15},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getIndexSizeInBits(0), V0) << Layout;
+    EXPECT_EQ(DL.getIndexSizeInBits(1), V1) << Layout;
+    EXPECT_EQ(DL.getIndexSizeInBits(2), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetIndexSize) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 8, 8, 8},
+      {"p:16:32", 2, 2, 2},
+      {"p0:27:64", 4, 4, 4},
+      {"p1:19:32:64:5", 8, 1, 8},
+      {"p1:33:32:64:23-p2:21:8:16:13", 8, 3, 2},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getIndexSize(0), V0) << Layout;
+    EXPECT_EQ(DL.getIndexSize(1), V1) << Layout;
+    EXPECT_EQ(DL.getIndexSize(2), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetPointerABIAlignment) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 8, 8, 8},
+      {"p:16:32", 4, 4, 4},
+      {"p0:16:32:64", 4, 4, 4},
+      {"p1:32:16:64", 8, 2, 8},
+      {"p1:33:16:32:15-p2:23:8:16:9", 8, 2, 1},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getPointerABIAlignment(0).value(), V0) << Layout;
+    EXPECT_EQ(DL.getPointerABIAlignment(1).value(), V1) << Layout;
+    EXPECT_EQ(DL.getPointerABIAlignment(2).value(), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetPointerPrefAlignment) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 8, 8, 8},
+      {"p:16:32", 4, 4, 4},
+      {"p0:8:16:32", 4, 4, 4},
+      {"p1:32:8:16", 8, 2, 8},
+      {"p1:33:8:16:31-p2:23:8:32:17", 8, 2, 4},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getPointerPrefAlignment(0).value(), V0) << Layout;
+    EXPECT_EQ(DL.getPointerPrefAlignment(1).value(), V1) << Layout;
+    EXPECT_EQ(DL.getPointerPrefAlignment(2).value(), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, IsNonIntegralAddressSpace) {
+  DataLayout Default;
+  EXPECT_THAT(Default.getNonIntegralAddressSpaces(), ::testing::SizeIs(0));
+  EXPECT_FALSE(Default.isNonIntegralAddressSpace(0));
+  EXPECT_FALSE(Default.isNonIntegralAddressSpace(1));
+
+  DataLayout Custom = cantFail(DataLayout::parse("ni:2:16777215"));
+  EXPECT_THAT(Custom.getNonIntegralAddressSpaces(),
+              ::testing::ElementsAreArray({2U, 16777215U}));
+  EXPECT_FALSE(Custom.isNonIntegralAddressSpace(0));
+  EXPECT_FALSE(Custom.isNonIntegralAddressSpace(1));
+  EXPECT_TRUE(Custom.isNonIntegralAddressSpace(2));
+  EXPECT_TRUE(Custom.isNonIntegralAddressSpace(16777215));
+}
+
 TEST(DataLayoutTest, CopyAssignmentInvalidatesStructLayout) {
   DataLayout DL1 = cantFail(DataLayout::parse("p:32:32"));
   DataLayout DL2 = cantFail(DataLayout::parse("p:64:64"));
diff --git a/llvm/unittests/Passes/PassBuilderBindings/PassBuilderBindingsTest.cpp b/llvm/unittests/Passes/PassBuilderBindings/PassBuilderBindingsTest.cpp
index 2b06033f0c3fa26..4e17b1ad09e2b35 100644
--- a/llvm/unittests/Passes/PassBuilderBindings/PassBuilderBindingsTest.cpp
+++ b/llvm/unittests/Passes/PassBuilderBindings/PassBuilderBindingsTest.cpp
@@ -35,6 +35,9 @@ class PassBuilderCTest : public testing::Test {
     LLVMDisposeMessage(Triple);
     Context = LLVMContextCreate();
     Module = LLVMModuleCreateWithNameInContext("test", Context);
+    LLVMTypeRef FT =
+        LLVMFunctionType(LLVMVoidTypeInContext(Context), nullptr, 0, 0);
+    Function = LLVMAddFunction(Module, "test", FT);
   }
 
   void TearDown() override {
@@ -52,6 +55,7 @@ class PassBuilderCTest : public testing::Test {
 public:
   LLVMTargetMachineRef TM;
   LLVMModuleRef Module;
+  LLVMValueRef Function;
   LLVMContextRef Context;
 };
 
@@ -63,7 +67,6 @@ TEST_F(PassBuilderCTest, Basic) {
   LLVMPassBuilderOptionsSetAAPipeline(Options, "basic-aa");
   if (LLVMErrorRef E = LLVMRunPasses(Module, "default<O2>", TM, Options)) {
     char *Msg = LLVMGetErrorMessage(E);
-    LLVMConsumeError(E);
     LLVMDisposePassBuilderOptions(Options);
     FAIL() << "Failed to run passes: " << Msg;
   }
@@ -80,3 +83,14 @@ TEST_F(PassBuilderCTest, InvalidPassIsError) {
   LLVMConsumeError(E2);
   LLVMDisposePassBuilderOptions(Options);
 }
+
+TEST_F(PassBuilderCTest, Function) {
+  LLVMPassBuilderOptionsRef Options = LLVMCreatePassBuilderOptions();
+  if (LLVMErrorRef E =
+          LLVMRunPassesOnFunction(Function, "no-op-function", TM, Options)) {
+    char *Msg = LLVMGetErrorMessage(E);
+    LLVMDisposePassBuilderOptions(Options);
+    FAIL() << "Failed to run passes on function: " << Msg;
+  }
+  LLVMDisposePassBuilderOptions(Options);
+}
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 1e4679ed6e802ed..1d6a26728c9c567 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -54,7 +54,6 @@ define void @foo(i32 %v1) {
   sandboxir::Argument *Arg0 = F->getArg(0);
   sandboxir::BasicBlock *BB = &*F->begin();
   sandboxir::Instruction *AddI = &*BB->begin();
-  sandboxir::OpaqueInst *OpaqueI = cast<sandboxir::OpaqueInst>(AddI);
   sandboxir::Constant *Const0 = cast<sandboxir::Constant>(Ctx.getValue(LLVMC));
 
   EXPECT_TRUE(isa<sandboxir::Function>(F));
@@ -62,42 +61,36 @@ define void @foo(i32 %v1) {
   EXPECT_FALSE(isa<sandboxir::Function>(BB));
   EXPECT_FALSE(isa<sandboxir::Function>(AddI));
   EXPECT_FALSE(isa<sandboxir::Function>(Const0));
-  EXPECT_FALSE(isa<sandboxir::Function>(OpaqueI));
 
   EXPECT_FALSE(isa<sandboxir::Argument>(F));
   EXPECT_TRUE(isa<sandboxir::Argument>(Arg0));
   EXPECT_FALSE(isa<sandboxir::Argument>(BB));
   EXPECT_FALSE(isa<sandboxir::Argument>(AddI));
   EXPECT_FALSE(isa<sandboxir::Argument>(Const0));
-  EXPECT_FALSE(isa<sandboxir::Argument>(OpaqueI));
 
   EXPECT_TRUE(isa<sandboxir::Constant>(F));
   EXPECT_FALSE(isa<sandboxir::Constant>(Arg0));
   EXPECT_FALSE(isa<sandboxir::Constant>(BB));
   EXPECT_FALSE(isa<sandboxir::Constant>(AddI));
   EXPECT_TRUE(isa<sandboxir::Constant>(Const0));
-  EXPECT_FALSE(isa<sandboxir::Constant>(OpaqueI));
 
   EXPECT_FALSE(isa<sandboxir::OpaqueInst>(F));
   EXPECT_FALSE(isa<sandboxir::OpaqueInst>(Arg0));
   EXPECT_FALSE(isa<sandboxir::OpaqueInst>(BB));
-  EXPECT_TRUE(isa<sandboxir::OpaqueInst>(AddI));
+  EXPECT_FALSE(isa<sandboxir::OpaqueInst>(AddI));
   EXPECT_FALSE(isa<sandboxir::OpaqueInst>(Const0));
-  EXPECT_TRUE(isa<sandboxir::OpaqueInst>(OpaqueI));
 
   EXPECT_FALSE(isa<sandboxir::Instruction>(F));
   EXPECT_FALSE(isa<sandboxir::Instruction>(Arg0));
   EXPECT_FALSE(isa<sandboxir::Instruction>(BB));
   EXPECT_TRUE(isa<sandboxir::Instruction>(AddI));
   EXPECT_FALSE(isa<sandboxir::Instruction>(Const0));
-  EXPECT_TRUE(isa<sandboxir::Instruction>(OpaqueI));
 
   EXPECT_TRUE(isa<sandboxir::User>(F));
   EXPECT_FALSE(isa<sandboxir::User>(Arg0));
   EXPECT_FALSE(isa<sandboxir::User>(BB));
   EXPECT_TRUE(isa<sandboxir::User>(AddI));
   EXPECT_TRUE(isa<sandboxir::User>(Const0));
-  EXPECT_TRUE(isa<sandboxir::User>(OpaqueI));
 
 #ifndef NDEBUG
   std::string Buff;
@@ -107,10 +100,33 @@ define void @foo(i32 %v1) {
   BB->dumpOS(BS);
   AddI->dumpOS(BS);
   Const0->dumpOS(BS);
-  OpaqueI->dumpOS(BS);
 #endif
 }
 
+TEST_F(SandboxIRTest, ConstantInt) {
+  parseIR(C, R"IR(
+define void @foo(i32 %v0) {
+  %add0 = add i32 %v0, 42
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto &BB = *F.begin();
+  auto It = BB.begin();
+  auto *Add0 = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *FortyTwo = cast<sandboxir::ConstantInt>(Add0->getOperand(1));
+
+  // Check that creating an identical constant gives us the same object.
+  auto *NewCI = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 42, Ctx);
+  EXPECT_EQ(NewCI, FortyTwo);
+  // Check new constant.
+  auto *FortyThree = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 43, Ctx);
+  EXPECT_NE(FortyThree, FortyTwo);
+}
+
 TEST_F(SandboxIRTest, Use) {
   parseIR(C, R"IR(
 define i32 @foo(i32 %v0, i32 %v1) {
@@ -183,7 +199,7 @@ define i32 @foo(i32 %v0, i32 %v1) {
   I0->getOperandUse(0).dumpOS(BS);
   EXPECT_EQ(Buff, R"IR(
 Def:  i32 %v0 ; SB2. (Argument)
-User:   %add0 = add i32 %v0, %v1 ; SB5. (Opaque)
+User:   %add0 = add i32 %v0, %v1 ; SB5. (BinaryOperator)
 OperandNo: 0
 )IR");
 #endif // NDEBUG
@@ -508,8 +524,8 @@ define void @foo(i8 %v1) {
   EXPECT_EQ(Ret->getIterator(), std::next(BB->begin(), 2));
 
   // Check getOpcode().
-  EXPECT_EQ(I0->getOpcode(), sandboxir::Instruction::Opcode::Opaque);
-  EXPECT_EQ(I1->getOpcode(), sandboxir::Instruction::Opcode::Opaque);
+  EXPECT_EQ(I0->getOpcode(), sandboxir::Instruction::Opcode::Add);
+  EXPECT_EQ(I1->getOpcode(), sandboxir::Instruction::Opcode::Sub);
   EXPECT_EQ(Ret->getOpcode(), sandboxir::Instruction::Opcode::Ret);
 
   // Check moveBefore(I).
@@ -618,12 +634,11 @@ define void @foo(i1 %c0, i8 %v0, i8 %v1, i1 %c1) {
   }
   {
     // Check SelectInst::create() Folded.
-    auto *False =
-        sandboxir::Constant::createInt(llvm::Type::getInt1Ty(C), 0, Ctx,
-                                       /*IsSigned=*/false);
+    auto *False = sandboxir::ConstantInt::get(llvm::Type::getInt1Ty(C), 0, Ctx,
+                                              /*IsSigned=*/false);
     auto *FortyTwo =
-        sandboxir::Constant::createInt(llvm::Type::getInt1Ty(C), 42, Ctx,
-                                       /*IsSigned=*/false);
+        sandboxir::ConstantInt::get(llvm::Type::getInt1Ty(C), 42, Ctx,
+                                    /*IsSigned=*/false);
     auto *NewSel =
         sandboxir::SelectInst::create(False, FortyTwo, FortyTwo, Ret, Ctx);
     EXPECT_TRUE(isa<sandboxir::Constant>(NewSel));
@@ -714,7 +729,7 @@ define void @foo(i8 %v0, i8 %v1, <2 x i8> %vec) {
 
   auto *LLVMArg0 = LLVMF.getArg(0);
   auto *LLVMArgVec = LLVMF.getArg(2);
-  auto *Zero = sandboxir::Constant::createInt(Type::getInt8Ty(C), 0, Ctx);
+  auto *Zero = sandboxir::ConstantInt::get(Type::getInt8Ty(C), 0, Ctx);
   auto *LLVMZero = llvm::ConstantInt::get(Type::getInt8Ty(C), 0);
   EXPECT_EQ(
       sandboxir::InsertElementInst::isValidOperands(ArgVec, Arg0, Zero),
@@ -1628,6 +1643,607 @@ define void @foo(i32 %arg, float %farg) {
   EXPECT_FALSE(FAdd->getFastMathFlags() != LLVMFAdd->getFastMathFlags());
 }
 
+TEST_F(SandboxIRTest, SwitchInst) {
+  parseIR(C, R"IR(
+define void @foo(i32 %cond0, i32 %cond1) {
+  entry:
+    switch i32 %cond0, label %default [ i32 0, label %bb0
+                                        i32 1, label %bb1 ]
+  bb0:
+    ret void
+  bb1:
+    ret void
+  default:
+    ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  auto *LLVMEntry = getBasicBlockByName(LLVMF, "entry");
+  auto *LLVMSwitch = cast<llvm::SwitchInst>(&*LLVMEntry->begin());
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *Cond1 = F.getArg(1);
+  auto *Entry = cast<sandboxir::BasicBlock>(Ctx.getValue(LLVMEntry));
+  auto *Switch = cast<sandboxir::SwitchInst>(&*Entry->begin());
+  auto *BB0 = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "bb0")));
+  auto *BB1 = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "bb1")));
+  auto *Default = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "default")));
+
+  // Check getCondition().
+  EXPECT_EQ(Switch->getCondition(), Ctx.getValue(LLVMSwitch->getCondition()));
+  // Check setCondition().
+  auto *OrigCond = Switch->getCondition();
+  auto *NewCond = Cond1;
+  EXPECT_NE(NewCond, OrigCond);
+  Switch->setCondition(NewCond);
+  EXPECT_EQ(Switch->getCondition(), NewCond);
+  Switch->setCondition(OrigCond);
+  EXPECT_EQ(Switch->getCondition(), OrigCond);
+  // Check getDefaultDest().
+  EXPECT_EQ(Switch->getDefaultDest(),
+            Ctx.getValue(LLVMSwitch->getDefaultDest()));
+  EXPECT_EQ(Switch->getDefaultDest(), Default);
+  // Check defaultDestUndefined().
+  EXPECT_EQ(Switch->defaultDestUndefined(), LLVMSwitch->defaultDestUndefined());
+  // Check setDefaultDest().
+  auto *OrigDefaultDest = Switch->getDefaultDest();
+  auto *NewDefaultDest = Entry;
+  EXPECT_NE(NewDefaultDest, OrigDefaultDest);
+  Switch->setDefaultDest(NewDefaultDest);
+  EXPECT_EQ(Switch->getDefaultDest(), NewDefaultDest);
+  Switch->setDefaultDest(OrigDefaultDest);
+  EXPECT_EQ(Switch->getDefaultDest(), OrigDefaultDest);
+  // Check getNumCases().
+  EXPECT_EQ(Switch->getNumCases(), LLVMSwitch->getNumCases());
+  // Check getNumSuccessors().
+  EXPECT_EQ(Switch->getNumSuccessors(), LLVMSwitch->getNumSuccessors());
+  // Check getSuccessor().
+  for (auto SuccIdx : seq<unsigned>(0, Switch->getNumSuccessors()))
+    EXPECT_EQ(Switch->getSuccessor(SuccIdx),
+              Ctx.getValue(LLVMSwitch->getSuccessor(SuccIdx)));
+  // Check setSuccessor().
+  auto *OrigSucc = Switch->getSuccessor(0);
+  auto *NewSucc = Entry;
+  EXPECT_NE(NewSucc, OrigSucc);
+  Switch->setSuccessor(0, NewSucc);
+  EXPECT_EQ(Switch->getSuccessor(0), NewSucc);
+  Switch->setSuccessor(0, OrigSucc);
+  EXPECT_EQ(Switch->getSuccessor(0), OrigSucc);
+  // Check case_begin(), case_end(), CaseIt.
+  auto *Zero = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 0, Ctx);
+  auto *One = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 1, Ctx);
+  auto CaseIt = Switch->case_begin();
+  {
+    sandboxir::SwitchInst::CaseHandle Case = *CaseIt++;
+    EXPECT_EQ(Case.getCaseValue(), Zero);
+    EXPECT_EQ(Case.getCaseSuccessor(), BB0);
+    EXPECT_EQ(Case.getCaseIndex(), 0u);
+    EXPECT_EQ(Case.getSuccessorIndex(), 1u);
+  }
+  {
+    sandboxir::SwitchInst::CaseHandle Case = *CaseIt++;
+    EXPECT_EQ(Case.getCaseValue(), One);
+    EXPECT_EQ(Case.getCaseSuccessor(), BB1);
+    EXPECT_EQ(Case.getCaseIndex(), 1u);
+    EXPECT_EQ(Case.getSuccessorIndex(), 2u);
+  }
+  EXPECT_EQ(CaseIt, Switch->case_end());
+  // Check cases().
+  unsigned CntCase = 0;
+  for (auto &Case : Switch->cases()) {
+    EXPECT_EQ(Case.getCaseIndex(), CntCase);
+    ++CntCase;
+  }
+  EXPECT_EQ(CntCase, 2u);
+  // Check case_default().
+  auto CaseDefault = *Switch->case_default();
+  EXPECT_EQ(CaseDefault.getCaseSuccessor(), Default);
+  EXPECT_EQ(CaseDefault.getCaseIndex(),
+            sandboxir::SwitchInst::DefaultPseudoIndex);
+  // Check findCaseValue().
+  EXPECT_EQ(Switch->findCaseValue(Zero)->getCaseIndex(), 0u);
+  EXPECT_EQ(Switch->findCaseValue(One)->getCaseIndex(), 1u);
+  // Check findCaseDest().
+  EXPECT_EQ(Switch->findCaseDest(BB0), Zero);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+  EXPECT_EQ(Switch->findCaseDest(Entry), nullptr);
+  // Check addCase().
+  auto *Two = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 2, Ctx);
+  Switch->addCase(Two, Entry);
+  auto CaseTwoIt = Switch->findCaseValue(Two);
+  auto CaseTwo = *CaseTwoIt;
+  EXPECT_EQ(CaseTwo.getCaseValue(), Two);
+  EXPECT_EQ(CaseTwo.getCaseSuccessor(), Entry);
+  EXPECT_EQ(Switch->getNumCases(), 3u);
+  // Check removeCase().
+  auto RemovedIt = Switch->removeCase(CaseTwoIt);
+  EXPECT_EQ(RemovedIt, Switch->case_end());
+  EXPECT_EQ(Switch->getNumCases(), 2u);
+  // Check create().
+  auto NewSwitch = sandboxir::SwitchInst::create(
+      Cond1, Default, 1, Default->begin(), Default, Ctx, "NewSwitch");
+  EXPECT_TRUE(isa<sandboxir::SwitchInst>(NewSwitch));
+  EXPECT_EQ(NewSwitch->getCondition(), Cond1);
+  EXPECT_EQ(NewSwitch->getDefaultDest(), Default);
+}
+
+TEST_F(SandboxIRTest, UnaryOperator) {
+  parseIR(C, R"IR(
+define void @foo(float %arg0) {
+  %fneg = fneg float %arg0
+  %copyfrom = fadd reassoc float %arg0, 42.0
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *Arg0 = F.getArg(0);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *I = cast<sandboxir::UnaryOperator>(&*It++);
+  auto *CopyFrom = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *Ret = &*It++;
+  EXPECT_EQ(I->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+  EXPECT_EQ(I->getOperand(0), Arg0);
+
+  {
+    // Check create() WhereIt, WhereBB.
+    auto *NewI =
+        cast<sandboxir::UnaryOperator>(sandboxir::UnaryOperator::create(
+            sandboxir::Instruction::Opcode::FNeg, Arg0,
+            /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+            "New1"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New1");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check create() InsertBefore.
+    auto *NewI =
+        cast<sandboxir::UnaryOperator>(sandboxir::UnaryOperator::create(
+            sandboxir::Instruction::Opcode::FNeg, Arg0,
+            /*InsertBefore=*/Ret, Ctx, "New2"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New2");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check create() InsertAtEnd.
+    auto *NewI =
+        cast<sandboxir::UnaryOperator>(sandboxir::UnaryOperator::create(
+            sandboxir::Instruction::Opcode::FNeg, Arg0,
+            /*InsertAtEnd=*/BB, Ctx, "New3"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New3");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getParent(), BB);
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+  }
+  {
+    // Check create() when it gets folded.
+    auto *FortyTwo = CopyFrom->getOperand(1);
+    auto *NewV = sandboxir::UnaryOperator::create(
+        sandboxir::Instruction::Opcode::FNeg, FortyTwo,
+        /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+        "Folded");
+    EXPECT_TRUE(isa<sandboxir::Constant>(NewV));
+  }
+
+  {
+    // Check createWithCopiedFlags() WhereIt, WhereBB.
+    auto *NewI = cast<sandboxir::UnaryOperator>(
+        sandboxir::UnaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::FNeg, Arg0, CopyFrom,
+            /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+            "NewCopyFrom1"));
+    EXPECT_EQ(NewI->hasAllowReassoc(), CopyFrom->hasAllowReassoc());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewCopyFrom1");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check createWithCopiedFlags() InsertBefore,
+    auto *NewI = cast<sandboxir::UnaryOperator>(
+        sandboxir::UnaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::FNeg, Arg0, CopyFrom,
+            /*InsertBefore=*/Ret, Ctx, "NewCopyFrom2"));
+    EXPECT_EQ(NewI->hasAllowReassoc(), CopyFrom->hasAllowReassoc());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewCopyFrom2");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check createWithCopiedFlags() InsertAtEnd,
+    auto *NewI = cast<sandboxir::UnaryOperator>(
+        sandboxir::UnaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::FNeg, Arg0, CopyFrom,
+            /*InsertAtEnd=*/BB, Ctx, "NewCopyFrom3"));
+    EXPECT_EQ(NewI->hasAllowReassoc(), CopyFrom->hasAllowReassoc());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewCopyFrom3");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getParent(), BB);
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+  }
+  {
+    // Check createWithCopiedFlags() when it gets folded.
+    auto *FortyTwo = CopyFrom->getOperand(1);
+    auto *NewV = sandboxir::UnaryOperator::createWithCopiedFlags(
+        sandboxir::Instruction::Opcode::FNeg, FortyTwo, CopyFrom,
+        /*InsertAtEnd=*/BB, Ctx, "Folded");
+    EXPECT_TRUE(isa<sandboxir::Constant>(NewV));
+  }
+}
+
+TEST_F(SandboxIRTest, BinaryOperator) {
+  parseIR(C, R"IR(
+define void @foo(i8 %arg0, i8 %arg1, float %farg0, float %farg1) {
+  %add = add i8 %arg0, %arg1
+  %fadd = fadd float %farg0, %farg1
+  %sub = sub i8 %arg0, %arg1
+  %fsub = fsub float %farg0, %farg1
+  %mul = mul i8 %arg0, %arg1
+  %fmul = fmul float %farg0, %farg1
+  %udiv = udiv i8 %arg0, %arg1
+  %sdiv = sdiv i8 %arg0, %arg1
+  %fdiv = fdiv float %farg0, %farg1
+  %urem = urem i8 %arg0, %arg1
+  %srem = srem i8 %arg0, %arg1
+  %frem = frem float %farg0, %farg1
+  %shl = shl i8 %arg0, %arg1
+  %lshr = lshr i8 %arg0, %arg1
+  %ashr = ashr i8 %arg0, %arg1
+  %and = and i8 %arg0, %arg1
+  %or = or i8 %arg0, %arg1
+  %xor = xor i8 %arg0, %arg1
+
+  %copyfrom = add nsw i8 %arg0, %arg1
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *Arg0 = F.getArg(0);
+  auto *Arg1 = F.getArg(1);
+  auto *FArg0 = F.getArg(2);
+  auto *FArg1 = F.getArg(3);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+
+#define CHECK_IBINOP(OPCODE)                                                   \
+  {                                                                            \
+    auto *I = cast<sandboxir::BinaryOperator>(&*It++);                         \
+    EXPECT_EQ(I->getOpcode(), OPCODE);                                         \
+    EXPECT_EQ(I->getOperand(0), Arg0);                                         \
+    EXPECT_EQ(I->getOperand(1), Arg1);                                         \
+  }
+#define CHECK_FBINOP(OPCODE)                                                   \
+  {                                                                            \
+    auto *I = cast<sandboxir::BinaryOperator>(&*It++);                         \
+    EXPECT_EQ(I->getOpcode(), OPCODE);                                         \
+    EXPECT_EQ(I->getOperand(0), FArg0);                                        \
+    EXPECT_EQ(I->getOperand(1), FArg1);                                        \
+  }
+
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Add);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FAdd);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Sub);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FSub);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Mul);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FMul);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::UDiv);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::SDiv);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FDiv);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::URem);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::SRem);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FRem);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Shl);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::LShr);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::AShr);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::And);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Or);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Xor);
+
+  auto *CopyFrom = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  {
+    // Check create() WhereIt, WhereBB.
+    auto *NewI =
+        cast<sandboxir::BinaryOperator>(sandboxir::BinaryOperator::create(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1,
+            /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+            "New1"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New1");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check create() InsertBefore.
+    auto *NewI =
+        cast<sandboxir::BinaryOperator>(sandboxir::BinaryOperator::create(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1,
+            /*InsertBefore=*/Ret, Ctx, "New2"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New2");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check create() InsertAtEnd.
+    auto *NewI =
+        cast<sandboxir::BinaryOperator>(sandboxir::BinaryOperator::create(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1,
+            /*InsertAtEnd=*/BB, Ctx, "New3"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New3");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+    EXPECT_EQ(NewI->getParent(), BB);
+  }
+  {
+    // Check create() when it gets folded.
+    auto *FortyTwo = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 42, Ctx);
+    auto *NewV = sandboxir::BinaryOperator::create(
+        sandboxir::Instruction::Opcode::Add, FortyTwo, FortyTwo,
+        /*InsertBefore=*/Ret, Ctx, "Folded");
+    EXPECT_TRUE(isa<sandboxir::Constant>(NewV));
+  }
+
+  {
+    // Check createWithCopiedFlags() WhereIt, WhereBB.
+    auto *NewI = cast<sandboxir::BinaryOperator>(
+        sandboxir::BinaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1, CopyFrom,
+            /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+            "NewNSW1"));
+    EXPECT_EQ(NewI->hasNoSignedWrap(), CopyFrom->hasNoSignedWrap());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewNSW1");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check createWithCopiedFlags() InsertBefore.
+    auto *NewI = cast<sandboxir::BinaryOperator>(
+        sandboxir::BinaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1, CopyFrom,
+            /*InsertBefore=*/Ret, Ctx, "NewNSW2"));
+    EXPECT_EQ(NewI->hasNoSignedWrap(), CopyFrom->hasNoSignedWrap());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewNSW2");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check createWithCopiedFlags() InsertAtEnd.
+    auto *NewI = cast<sandboxir::BinaryOperator>(
+        sandboxir::BinaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1, CopyFrom,
+            /*InsertAtEnd=*/BB, Ctx, "NewNSW3"));
+    EXPECT_EQ(NewI->hasNoSignedWrap(), CopyFrom->hasNoSignedWrap());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewNSW3");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getParent(), BB);
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+  }
+  {
+    // Check createWithCopiedFlags() when it gets folded.
+    auto *FortyTwo = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 42, Ctx);
+    auto *NewV = sandboxir::BinaryOperator::createWithCopiedFlags(
+        sandboxir::Instruction::Opcode::Add, FortyTwo, FortyTwo, CopyFrom,
+        /*InsertBefore=*/Ret, Ctx, "Folded");
+    EXPECT_TRUE(isa<sandboxir::Constant>(NewV));
+  }
+}
+
+TEST_F(SandboxIRTest, AtomicRMWInst) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %arg) {
+  %atomicrmw = atomicrmw add ptr %ptr, i8 %arg acquire, align 128
+  ret void
+}
+)IR");
+  llvm::Function &LLVMF = *M->getFunction("foo");
+  llvm::BasicBlock *LLVMBB = &*LLVMF.begin();
+  auto LLVMIt = LLVMBB->begin();
+  auto *LLVMRMW = cast<llvm::AtomicRMWInst>(&*LLVMIt++);
+
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(&LLVMF);
+  auto *Ptr = F->getArg(0);
+  auto *Arg = F->getArg(1);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *RMW = cast<sandboxir::AtomicRMWInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  // Check getOperationName().
+  EXPECT_EQ(
+      sandboxir::AtomicRMWInst::getOperationName(
+          sandboxir::AtomicRMWInst::BinOp::Add),
+      llvm::AtomicRMWInst::getOperationName(llvm::AtomicRMWInst::BinOp::Add));
+  // Check isFPOperation().
+  EXPECT_EQ(
+      sandboxir::AtomicRMWInst::isFPOperation(
+          sandboxir::AtomicRMWInst::BinOp::Add),
+      llvm::AtomicRMWInst::isFPOperation(llvm::AtomicRMWInst::BinOp::Add));
+  EXPECT_FALSE(sandboxir::AtomicRMWInst::isFPOperation(
+      sandboxir::AtomicRMWInst::BinOp::Add));
+  EXPECT_TRUE(sandboxir::AtomicRMWInst::isFPOperation(
+      sandboxir::AtomicRMWInst::BinOp::FAdd));
+  // Check setOperation(), getOperation().
+  EXPECT_EQ(RMW->getOperation(), LLVMRMW->getOperation());
+  RMW->setOperation(sandboxir::AtomicRMWInst::BinOp::Sub);
+  EXPECT_EQ(RMW->getOperation(), sandboxir::AtomicRMWInst::BinOp::Sub);
+  RMW->setOperation(sandboxir::AtomicRMWInst::BinOp::Add);
+  // Check getAlign().
+  EXPECT_EQ(RMW->getAlign(), LLVMRMW->getAlign());
+  auto OrigAlign = RMW->getAlign();
+  Align NewAlign(256);
+  EXPECT_NE(NewAlign, OrigAlign);
+  RMW->setAlignment(NewAlign);
+  EXPECT_EQ(RMW->getAlign(), NewAlign);
+  RMW->setAlignment(OrigAlign);
+  EXPECT_EQ(RMW->getAlign(), OrigAlign);
+  // Check isVolatile(), setVolatile().
+  EXPECT_EQ(RMW->isVolatile(), LLVMRMW->isVolatile());
+  bool OrigV = RMW->isVolatile();
+  bool NewV = true;
+  EXPECT_NE(NewV, OrigV);
+  RMW->setVolatile(NewV);
+  EXPECT_EQ(RMW->isVolatile(), NewV);
+  RMW->setVolatile(OrigV);
+  EXPECT_EQ(RMW->isVolatile(), OrigV);
+  // Check getOrdering(), setOrdering().
+  EXPECT_EQ(RMW->getOrdering(), LLVMRMW->getOrdering());
+  auto OldOrdering = RMW->getOrdering();
+  auto NewOrdering = AtomicOrdering::Monotonic;
+  EXPECT_NE(NewOrdering, OldOrdering);
+  RMW->setOrdering(NewOrdering);
+  EXPECT_EQ(RMW->getOrdering(), NewOrdering);
+  RMW->setOrdering(OldOrdering);
+  EXPECT_EQ(RMW->getOrdering(), OldOrdering);
+  // Check getSyncScopeID(), setSyncScopeID().
+  EXPECT_EQ(RMW->getSyncScopeID(), LLVMRMW->getSyncScopeID());
+  auto OrigSSID = RMW->getSyncScopeID();
+  SyncScope::ID NewSSID = SyncScope::SingleThread;
+  EXPECT_NE(NewSSID, OrigSSID);
+  RMW->setSyncScopeID(NewSSID);
+  EXPECT_EQ(RMW->getSyncScopeID(), NewSSID);
+  RMW->setSyncScopeID(OrigSSID);
+  EXPECT_EQ(RMW->getSyncScopeID(), OrigSSID);
+  // Check getPointerOperand().
+  EXPECT_EQ(RMW->getPointerOperand(),
+            Ctx.getValue(LLVMRMW->getPointerOperand()));
+  // Check getValOperand().
+  EXPECT_EQ(RMW->getValOperand(), Ctx.getValue(LLVMRMW->getValOperand()));
+  // Check getPointerAddressSpace().
+  EXPECT_EQ(RMW->getPointerAddressSpace(), LLVMRMW->getPointerAddressSpace());
+  // Check isFloatingPointOperation().
+  EXPECT_EQ(RMW->isFloatingPointOperation(),
+            LLVMRMW->isFloatingPointOperation());
+
+  Align Align(1024);
+  auto Ordering = AtomicOrdering::Acquire;
+  auto SSID = SyncScope::System;
+  {
+    // Check create() WhereIt, WhereBB.
+    auto *NewI =
+        cast<sandboxir::AtomicRMWInst>(sandboxir::AtomicRMWInst::create(
+            sandboxir::AtomicRMWInst::BinOp::Sub, Ptr, Arg, Align, Ordering,
+            /*WhereIt=*/Ret->getIterator(),
+            /*WhereBB=*/Ret->getParent(), Ctx, SSID, "NewAtomicRMW1"));
+    // Check getOpcode().
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::AtomicRMW);
+    // Check getAlign().
+    EXPECT_EQ(NewI->getAlign(), Align);
+    // Check getSuccessOrdering().
+    EXPECT_EQ(NewI->getOrdering(), Ordering);
+    // Check instr position.
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+    // Check getPointerOperand().
+    EXPECT_EQ(NewI->getPointerOperand(), Ptr);
+    // Check getValOperand().
+    EXPECT_EQ(NewI->getValOperand(), Arg);
+#ifndef NDEBUG
+    // Check getName().
+    EXPECT_EQ(NewI->getName(), "NewAtomicRMW1");
+#endif // NDEBUG
+  }
+  {
+    // Check create() InsertBefore.
+    auto *NewI =
+        cast<sandboxir::AtomicRMWInst>(sandboxir::AtomicRMWInst::create(
+            sandboxir::AtomicRMWInst::BinOp::Sub, Ptr, Arg, Align, Ordering,
+            /*InsertBefore=*/Ret, Ctx, SSID, "NewAtomicRMW2"));
+    // Check getOpcode().
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::AtomicRMW);
+    // Check getAlign().
+    EXPECT_EQ(NewI->getAlign(), Align);
+    // Check getSuccessOrdering().
+    EXPECT_EQ(NewI->getOrdering(), Ordering);
+    // Check instr position.
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+    // Check getPointerOperand().
+    EXPECT_EQ(NewI->getPointerOperand(), Ptr);
+    // Check getValOperand().
+    EXPECT_EQ(NewI->getValOperand(), Arg);
+#ifndef NDEBUG
+    // Check getName().
+    EXPECT_EQ(NewI->getName(), "NewAtomicRMW2");
+#endif // NDEBUG
+  }
+  {
+    // Check create() InsertAtEnd.
+    auto *NewI =
+        cast<sandboxir::AtomicRMWInst>(sandboxir::AtomicRMWInst::create(
+            sandboxir::AtomicRMWInst::BinOp::Sub, Ptr, Arg, Align, Ordering,
+            /*InsertAtEnd=*/BB, Ctx, SSID, "NewAtomicRMW3"));
+    // Check getOpcode().
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::AtomicRMW);
+    // Check getAlign().
+    EXPECT_EQ(NewI->getAlign(), Align);
+    // Check getSuccessOrdering().
+    EXPECT_EQ(NewI->getOrdering(), Ordering);
+    // Check instr position.
+    EXPECT_EQ(NewI->getParent(), BB);
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+    // Check getPointerOperand().
+    EXPECT_EQ(NewI->getPointerOperand(), Ptr);
+    // Check getValOperand().
+    EXPECT_EQ(NewI->getValOperand(), Arg);
+#ifndef NDEBUG
+    // Check getName().
+    EXPECT_EQ(NewI->getName(), "NewAtomicRMW3");
+#endif // NDEBUG
+  }
+}
+
 TEST_F(SandboxIRTest, AtomicCmpXchgInst) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr, i8 %cmp, i8 %new) {
@@ -1911,7 +2527,7 @@ define void @foo() {
   auto *Ty = Type::getInt32Ty(C);
   unsigned AddrSpace = 42;
   auto *PtrTy = PointerType::get(C, AddrSpace);
-  auto *ArraySize = sandboxir::Constant::createInt(Ty, 43, Ctx);
+  auto *ArraySize = sandboxir::ConstantInt::get(Ty, 43, Ctx);
   {
     // Check create() WhereIt, WhereBB.
     auto *NewI = cast<sandboxir::AllocaInst>(sandboxir::AllocaInst::create(
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index c1f23c95cbfaed0..b6770b237853cc1 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -644,6 +644,141 @@ define void @foo(i8 %arg) {
   EXPECT_EQ(Invoke->getSuccessor(1), ExceptionBB);
 }
 
+TEST_F(TrackerTest, SwitchInstSetters) {
+  parseIR(C, R"IR(
+define void @foo(i32 %cond0, i32 %cond1) {
+  entry:
+    switch i32 %cond0, label %default [ i32 0, label %bb0
+                                        i32 1, label %bb1 ]
+  bb0:
+    ret void
+  bb1:
+    ret void
+  default:
+    ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  auto *LLVMEntry = getBasicBlockByName(LLVMF, "entry");
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *Cond1 = F.getArg(1);
+  auto *Entry = cast<sandboxir::BasicBlock>(Ctx.getValue(LLVMEntry));
+  auto *BB0 = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "bb0")));
+  auto *BB1 = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "bb1")));
+  auto *Switch = cast<sandboxir::SwitchInst>(&*Entry->begin());
+
+  // Check setCondition().
+  auto *OrigCond = Switch->getCondition();
+  auto *NewCond = Cond1;
+  EXPECT_NE(NewCond, OrigCond);
+  Ctx.save();
+  Switch->setCondition(NewCond);
+  EXPECT_EQ(Switch->getCondition(), NewCond);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getCondition(), OrigCond);
+  // Check setDefaultDest().
+  auto *OrigDefaultDest = Switch->getDefaultDest();
+  auto *NewDefaultDest = Entry;
+  EXPECT_NE(NewDefaultDest, OrigDefaultDest);
+  Ctx.save();
+  Switch->setDefaultDest(NewDefaultDest);
+  EXPECT_EQ(Switch->getDefaultDest(), NewDefaultDest);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getDefaultDest(), OrigDefaultDest);
+  // Check setSuccessor().
+  auto *OrigSucc = Switch->getSuccessor(0);
+  auto *NewSucc = Entry;
+  EXPECT_NE(NewSucc, OrigSucc);
+  Ctx.save();
+  Switch->setSuccessor(0, NewSucc);
+  EXPECT_EQ(Switch->getSuccessor(0), NewSucc);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getSuccessor(0), OrigSucc);
+  // Check addCase().
+  auto *Zero = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 0, Ctx);
+  auto *One = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 1, Ctx);
+  auto *FortyTwo = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 42, Ctx);
+  Ctx.save();
+  Switch->addCase(FortyTwo, Entry);
+  EXPECT_EQ(Switch->getNumCases(), 3u);
+  EXPECT_EQ(Switch->findCaseDest(Entry), FortyTwo);
+  EXPECT_EQ(Switch->findCaseValue(FortyTwo)->getCaseSuccessor(), Entry);
+  EXPECT_EQ(Switch->findCaseDest(BB0), Zero);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getNumCases(), 2u);
+  EXPECT_EQ(Switch->findCaseDest(BB0), Zero);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+  // Check removeCase().
+  Ctx.save();
+  Switch->removeCase(Switch->findCaseValue(Zero));
+  EXPECT_EQ(Switch->getNumCases(), 1u);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getNumCases(), 2u);
+  EXPECT_EQ(Switch->findCaseDest(BB0), Zero);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+}
+
+TEST_F(TrackerTest, AtomicRMWSetters) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %arg) {
+  %atomicrmw = atomicrmw add ptr %ptr, i8 %arg acquire, align 128
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *RMW = cast<sandboxir::AtomicRMWInst>(&*It++);
+
+  // Check setAlignment().
+  Ctx.save();
+  auto OrigAlign = RMW->getAlign();
+  Align NewAlign(1024);
+  EXPECT_NE(NewAlign, OrigAlign);
+  RMW->setAlignment(NewAlign);
+  EXPECT_EQ(RMW->getAlign(), NewAlign);
+  Ctx.revert();
+  EXPECT_EQ(RMW->getAlign(), OrigAlign);
+
+  // Check setVolatile().
+  Ctx.save();
+  auto OrigIsVolatile = RMW->isVolatile();
+  bool NewIsVolatile = true;
+  EXPECT_NE(NewIsVolatile, OrigIsVolatile);
+  RMW->setVolatile(NewIsVolatile);
+  EXPECT_EQ(RMW->isVolatile(), NewIsVolatile);
+  Ctx.revert();
+  EXPECT_EQ(RMW->isVolatile(), OrigIsVolatile);
+
+  // Check setOrdering().
+  Ctx.save();
+  auto OrigOrdering = RMW->getOrdering();
+  auto NewOrdering = AtomicOrdering::SequentiallyConsistent;
+  EXPECT_NE(NewOrdering, OrigOrdering);
+  RMW->setOrdering(NewOrdering);
+  EXPECT_EQ(RMW->getOrdering(), NewOrdering);
+  Ctx.revert();
+  EXPECT_EQ(RMW->getOrdering(), OrigOrdering);
+
+  // Check setSyncScopeID().
+  Ctx.save();
+  auto OrigSSID = RMW->getSyncScopeID();
+  auto NewSSID = SyncScope::SingleThread;
+  EXPECT_NE(NewSSID, OrigSSID);
+  RMW->setSyncScopeID(NewSSID);
+  EXPECT_EQ(RMW->getSyncScopeID(), NewSSID);
+  Ctx.revert();
+  EXPECT_EQ(RMW->getSyncScopeID(), OrigSSID);
+}
+
 TEST_F(TrackerTest, AtomicCmpXchgSetters) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr, i8 %cmp, i8 %new) {
diff --git a/llvm/unittests/Support/MathExtrasTest.cpp b/llvm/unittests/Support/MathExtrasTest.cpp
index a1cc609033d972a..984185fece4aa3c 100644
--- a/llvm/unittests/Support/MathExtrasTest.cpp
+++ b/llvm/unittests/Support/MathExtrasTest.cpp
@@ -218,8 +218,11 @@ TEST(MathExtras, AlignToPowerOf2) {
   EXPECT_EQ(24u, alignToPowerOf2(17, 8));
   EXPECT_EQ(0u, alignToPowerOf2(~0LL, 8));
   EXPECT_EQ(240u, alignToPowerOf2(240, 16));
-  EXPECT_EQ(static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1,
-            alignToPowerOf2(std::numeric_limits<uint32_t>::max(), 2));
+
+  // Overflow.
+  EXPECT_EQ(0u, alignToPowerOf2(static_cast<uint8_t>(200),
+                                static_cast<uint8_t>(128)));
+  EXPECT_EQ(0u, alignToPowerOf2(std::numeric_limits<uint32_t>::max(), 2));
 }
 
 TEST(MathExtras, AlignDown) {
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index bede4e64696c55c..d1de96477a5ca06 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -620,6 +620,94 @@ TEST(ParseArchString, RejectsConflictingExtensions) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
               "'zcf' is only supported for 'rv32'");
   }
+
+  for (StringRef Input : {"rv64i_xwchc"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'xwchc' is only supported for 'rv32'");
+  }
+
+  for (StringRef Input : {"rv32id_xwchc"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'d' and 'xwchc' extensions are incompatible");
+  }
+
+  for (StringRef Input : {"rv32i_zcb_xwchc"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'xwchc' and 'zcb' extensions are incompatible");
+  }
+}
+
+TEST(ParseArchString, MissingDepency) {
+  for (StringRef Input : {"rv32i_zvl32b", "rv64i_zvl128b"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvl*b' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvbb"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvbb' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvbc32e0p7"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvbc32e' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvbc"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvbc' requires 'v' or 'zve64*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvkb"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvkb' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvkg"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvkg' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvkgs0p7"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvkg' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvkned"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvkned' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvknha"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvknha' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvksed"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvksed' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvksh"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvksh' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvknhb"}) {
+    EXPECT_EQ(
+        toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+        "'zvknhb' requires 'v' or 'zve64*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zacas1p0"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zacas' requires 'a' or 'zaamo' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zabha"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zabha' requires 'a' or 'zaamo' extension to also be specified");
+  }
 }
 
 TEST(ParseArchString, RejectsUnrecognizedProfileNames) {
@@ -1029,6 +1117,8 @@ Experimental extensions
     zicfiss              1.0
     zacas                1.0
     zalasr               0.1
+    zvbc32e              0.7
+    zvkgs                0.7
     smmpm                1.0
     smnpm                1.0
     ssnpm                1.0
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 960a9892202b35e..3d99ea8d9196288 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1141,7 +1141,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_FP16,        AArch64::AEK_FP16FML,
                               AArch64::AEK_SB,          AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,         AArch64::AEK_AM}),
+                              AArch64::AEK_ETE,         AArch64::AEK_AM,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a520", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1156,7 +1157,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PERFMON,     AArch64::AEK_PREDRES,
                               AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
                               AArch64::AEK_PERFMON,     AArch64::AEK_AM,
-                              AArch64::AEK_ETE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a520ae", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1171,7 +1172,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PERFMON,     AArch64::AEK_PREDRES,
                               AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
                               AArch64::AEK_PERFMON,     AArch64::AEK_AM,
-                              AArch64::AEK_ETE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a57", "armv8-a",
                              {AArch64::AEK_CRC, AArch64::AEK_AES,
                               AArch64::AEK_SHA2, AArch64::AEK_FP,
@@ -1256,7 +1257,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SVE2,    AArch64::AEK_SVE2BITPERM, AArch64::AEK_PAUTH,
              AArch64::AEK_FLAGM,   AArch64::AEK_SB,          AArch64::AEK_I8MM,
              AArch64::AEK_BF16,    AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
-             AArch64::AEK_PERFMON, AArch64::AEK_ETE}),
+             AArch64::AEK_PERFMON, AArch64::AEK_ETE,         AArch64::AEK_CCIDX,
+             AArch64::AEK_SSBS}),
         AArch64CPUTestParams("cortex-a715", "armv9-a",
                              {AArch64::AEK_CRC,     AArch64::AEK_FP,
                               AArch64::AEK_BF16,    AArch64::AEK_SIMD,
@@ -1271,7 +1273,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_FP16FML, AArch64::AEK_FP16,
                               AArch64::AEK_FLAGM,   AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,    AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,     AArch64::AEK_TRBE}),
+                              AArch64::AEK_ETE,     AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a720", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1287,7 +1290,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_TRBE}),
+                              AArch64::AEK_TRBE,        AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a720ae", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1303,7 +1306,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_TRBE}),
+                              AArch64::AEK_TRBE,        AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a725", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1318,7 +1321,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PERFMON,     AArch64::AEK_PREDRES,
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_ETE,
-                              AArch64::AEK_SPE_EEF,     AArch64::AEK_TRBE}),
+                              AArch64::AEK_SPE_EEF,     AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "neoverse-v1", "armv8.4-a",
             {AArch64::AEK_RAS,     AArch64::AEK_SVE,     AArch64::AEK_SSBS,
@@ -1329,7 +1333,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SM4,     AArch64::AEK_FP16,    AArch64::AEK_BF16,
              AArch64::AEK_PROFILE, AArch64::AEK_RAND,    AArch64::AEK_FP16FML,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH,   AArch64::AEK_PERFMON, AArch64::AEK_CCDP}),
+             AArch64::AEK_PAUTH,   AArch64::AEK_PERFMON, AArch64::AEK_CCDP,
+             AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("neoverse-v2", "armv9-a",
                              {AArch64::AEK_RAS,         AArch64::AEK_SVE,
                               AArch64::AEK_SSBS,        AArch64::AEK_RCPC,
@@ -1343,7 +1348,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_SVE2BITPERM, AArch64::AEK_RAND,
                               AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
                               AArch64::AEK_PAUTH,       AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("neoverse-v3", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1361,7 +1366,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_RME}),
+                              AArch64::AEK_RME,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("neoverse-v3ae", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1379,7 +1384,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_RME}),
+                              AArch64::AEK_RME,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "cortex-r82", "armv8-r",
             {AArch64::AEK_CRC, AArch64::AEK_RDM, AArch64::AEK_SSBS,
@@ -1427,7 +1432,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_FP16FML,     AArch64::AEK_FLAGM,
                               AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
                               AArch64::AEK_PERFMON,     AArch64::AEK_AM,
-                              AArch64::AEK_ETE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-x3", "armv9-a",
                              {AArch64::AEK_CRC,     AArch64::AEK_FP,
                               AArch64::AEK_BF16,    AArch64::AEK_SIMD,
@@ -1442,7 +1447,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PREDRES, AArch64::AEK_FLAGM,
                               AArch64::AEK_SSBS,    AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,    AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,     AArch64::AEK_TRBE}),
+                              AArch64::AEK_ETE,     AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-x4", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1458,7 +1464,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_TRBE}),
+                              AArch64::AEK_TRBE,        AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-x925", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1473,7 +1479,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PERFMON,     AArch64::AEK_PREDRES,
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_ETE,
-                              AArch64::AEK_SPE_EEF,     AArch64::AEK_TRBE}),
+                              AArch64::AEK_SPE_EEF,     AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cyclone", "armv8-a",
                              {AArch64::AEK_AES, AArch64::AEK_SHA2,
                               AArch64::AEK_FP, AArch64::AEK_SIMD,
@@ -1591,7 +1598,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD, AArch64::AEK_FP16,
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON}),
+             AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
+             AArch64::AEK_SSBS}),
         AArch64CPUTestParams(
             "apple-m2", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_AES,     AArch64::AEK_SHA2,
@@ -1600,7 +1608,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD, AArch64::AEK_FP16,
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON}),
+             AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
+             AArch64::AEK_SSBS}),
         AArch64CPUTestParams(
             "apple-a16", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_AES,     AArch64::AEK_SHA2,
@@ -1610,7 +1619,7 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
              AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
-             AArch64::AEK_HCX}),
+             AArch64::AEK_HCX,     AArch64::AEK_SSBS}),
         AArch64CPUTestParams(
             "apple-m3", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_AES,     AArch64::AEK_SHA2,
@@ -1620,7 +1629,7 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
              AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
-             AArch64::AEK_HCX}),
+             AArch64::AEK_HCX,     AArch64::AEK_SSBS}),
         AArch64CPUTestParams(
             "apple-a17", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_AES,     AArch64::AEK_SHA2,
@@ -1630,7 +1639,7 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
              AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
-             AArch64::AEK_HCX}),
+             AArch64::AEK_HCX,     AArch64::AEK_SSBS}),
         AArch64CPUTestParams("apple-m4", "armv9.2-a",
                              {AArch64::AEK_CRC,       AArch64::AEK_AES,
                               AArch64::AEK_SHA2,      AArch64::AEK_SHA3,
@@ -1695,7 +1704,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_I8MM,        AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PAUTH,
                               AArch64::AEK_FP16FML,     AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,         AArch64::AEK_TRBE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("neoverse-n3", "armv9.2-a",
                              {AArch64::AEK_BF16,    AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,     AArch64::AEK_SVE2,
@@ -1711,7 +1721,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_RAND,    AArch64::AEK_SVE2BITPERM,
                               AArch64::AEK_FP16FML, AArch64::AEK_PROFILE,
                               AArch64::AEK_JSCVT,   AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,     AArch64::AEK_SPE_EEF}),
+                              AArch64::AEK_ETE,     AArch64::AEK_SPE_EEF,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "ampere1", "armv8.6-a",
             {AArch64::AEK_CRC,  AArch64::AEK_FP,    AArch64::AEK_FP16,
@@ -1720,17 +1731,18 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SHA3, AArch64::AEK_BF16,  AArch64::AEK_SHA2,
              AArch64::AEK_AES,  AArch64::AEK_I8MM,  AArch64::AEK_SSBS,
              AArch64::AEK_SB,   AArch64::AEK_RAND,  AArch64::AEK_JSCVT,
-             AArch64::AEK_FCMA, AArch64::AEK_PAUTH, AArch64::AEK_PERFMON}),
+             AArch64::AEK_FCMA, AArch64::AEK_PAUTH, AArch64::AEK_PERFMON,
+             AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "ampere1a", "armv8.6-a",
-            {AArch64::AEK_CRC,   AArch64::AEK_FP,     AArch64::AEK_FP16,
-             AArch64::AEK_SIMD,  AArch64::AEK_RAS,    AArch64::AEK_LSE,
-             AArch64::AEK_RDM,   AArch64::AEK_RCPC,   AArch64::AEK_DOTPROD,
-             AArch64::AEK_SM4,   AArch64::AEK_SHA3,   AArch64::AEK_BF16,
-             AArch64::AEK_SHA2,  AArch64::AEK_AES,    AArch64::AEK_I8MM,
-             AArch64::AEK_SSBS,  AArch64::AEK_SB,     AArch64::AEK_RAND,
-             AArch64::AEK_MTE,   AArch64::AEK_JSCVT,  AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH, AArch64::AEK_PERFMON}),
+            {AArch64::AEK_CRC,   AArch64::AEK_FP,      AArch64::AEK_FP16,
+             AArch64::AEK_SIMD,  AArch64::AEK_RAS,     AArch64::AEK_LSE,
+             AArch64::AEK_RDM,   AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD,
+             AArch64::AEK_SM4,   AArch64::AEK_SHA3,    AArch64::AEK_BF16,
+             AArch64::AEK_SHA2,  AArch64::AEK_AES,     AArch64::AEK_I8MM,
+             AArch64::AEK_SSBS,  AArch64::AEK_SB,      AArch64::AEK_RAND,
+             AArch64::AEK_MTE,   AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
+             AArch64::AEK_PAUTH, AArch64::AEK_PERFMON, AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "ampere1b", "armv8.7-a",
             {AArch64::AEK_CRC,   AArch64::AEK_FP,    AArch64::AEK_FP16,
@@ -1741,7 +1753,7 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SSBS,  AArch64::AEK_SB,    AArch64::AEK_RAND,
              AArch64::AEK_MTE,   AArch64::AEK_JSCVT, AArch64::AEK_FCMA,
              AArch64::AEK_PAUTH, AArch64::AEK_CSSC,  AArch64::AEK_PERFMON,
-             AArch64::AEK_WFXT}),
+             AArch64::AEK_WFXT,  AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "neoverse-512tvb", "armv8.4-a",
             {AArch64::AEK_RAS,     AArch64::AEK_SVE,     AArch64::AEK_SSBS,
@@ -1752,7 +1764,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SM4,     AArch64::AEK_FP16,    AArch64::AEK_BF16,
              AArch64::AEK_PROFILE, AArch64::AEK_RAND,    AArch64::AEK_FP16FML,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH,   AArch64::AEK_PERFMON, AArch64::AEK_CCDP}),
+             AArch64::AEK_PAUTH,   AArch64::AEK_PERFMON, AArch64::AEK_CCDP,
+             AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("thunderx2t99", "armv8.1-a",
                              {AArch64::AEK_CRC, AArch64::AEK_AES,
                               AArch64::AEK_SHA2, AArch64::AEK_LSE,
@@ -1765,7 +1778,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_SIMD, AArch64::AEK_RAS,
                               AArch64::AEK_RCPC, AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA, AArch64::AEK_PAUTH,
-                              AArch64::AEK_PERFMON}),
+                              AArch64::AEK_PERFMON, AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("thunderx", "armv8-a",
                              {AArch64::AEK_CRC, AArch64::AEK_AES,
                               AArch64::AEK_SHA2, AArch64::AEK_SIMD,
@@ -1808,7 +1821,7 @@ INSTANTIATE_TEST_SUITE_P(
             {AArch64::AEK_AES, AArch64::AEK_FP, AArch64::AEK_SIMD,
              AArch64::AEK_PERFMON, AArch64::AEK_SHA2, AArch64::AEK_PROFILE,
              AArch64::AEK_CRC, AArch64::AEK_LSE, AArch64::AEK_RDM,
-             AArch64::AEK_RAS, AArch64::AEK_RCPC}),
+             AArch64::AEK_RAS, AArch64::AEK_RCPC, AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "oryon-1", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_FP,      AArch64::AEK_PAUTH,
@@ -1817,7 +1830,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD, AArch64::AEK_SM4,
              AArch64::AEK_SHA3,    AArch64::AEK_BF16,    AArch64::AEK_SHA2,
              AArch64::AEK_AES,     AArch64::AEK_I8MM,    AArch64::AEK_RAND,
-             AArch64::AEK_PROFILE, AArch64::AEK_PERFMON})),
+             AArch64::AEK_PROFILE, AArch64::AEK_PERFMON, AArch64::AEK_CCIDX,
+             AArch64::AEK_SSBS})),
 
     AArch64CPUTestParams::PrintToStringParamName);
 
@@ -2248,9 +2262,9 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
 }
 
 TEST(TargetParserTest, AArch64PrintSupportedExtensions) {
-  std::string expected =
-      "All available -march extensions for AArch64\n\n"
-      "    Name                Architecture Feature(s)                                Description\n";
+  std::string expected = "All available -march extensions for AArch64\n\n"
+                         "    Name                Architecture Feature(s)      "
+                         "                          Description\n";
 
   outs().flush();
   testing::internal::CaptureStdout();
diff --git a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
index a4b25025b3c618a..71ca331461c00c8 100644
--- a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
@@ -19,10 +19,38 @@
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cstdint>
+#include <set>
 #include <string>
 
 using namespace llvm;
 
+/// Collect the full set of implied features for a SubtargetFeature.
+static void CollectImpliedFeatures(std::set<Record *> &SeenFeats, Record *Rec) {
+  assert(Rec->isSubClassOf("SubtargetFeature") &&
+         "Rec is not a SubtargetFeature");
+
+  SeenFeats.insert(Rec);
+  for (Record *Implied : Rec->getValueAsListOfDefs("Implies"))
+    CollectImpliedFeatures(SeenFeats, Implied);
+}
+
+static void CheckFeatureTree(Record *Root) {
+  std::set<Record *> SeenFeats;
+  CollectImpliedFeatures(SeenFeats, Root);
+
+  // Check that each of the mandatory (implied) features which is an
+  // ExtensionWithMArch is also enabled by default.
+  auto DefaultExtsVec = Root->getValueAsListOfDefs("DefaultExts");
+  std::set<Record *> DefaultExts{DefaultExtsVec.begin(), DefaultExtsVec.end()};
+  for (auto *Feat : SeenFeats) {
+    if (Feat->isSubClassOf("ExtensionWithMArch") && !DefaultExts.count(Feat))
+      PrintFatalError(Root->getLoc(),
+                      "ExtensionWithMArch " + Feat->getName() +
+                          " is implied (mandatory) as a SubtargetFeature, but "
+                          "is not present in DefaultExts");
+  }
+}
+
 static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
   OS << "// Autogenerated by ARMTargetDefEmitter.cpp\n\n";
 
@@ -283,9 +311,7 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
     auto Profile = Arch->getValueAsString("Profile");
     auto ArchInfo = ArchInfoName(Major, Minor, Profile);
 
-    // The apple-latest alias is backend only, do not expose it to -mcpu.
-    if (Name == "apple-latest")
-      continue;
+    CheckFeatureTree(Arch);
 
     OS << "  {\n"
        << "    \"" << Name << "\",\n"
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
index 9153645e3d9a117..4bca904e9f38bdf 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
@@ -29,16 +29,16 @@ CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) {
   std::vector<Record *> IntrProperties =
       RC.getAllDerivedDefinitions("IntrinsicProperty");
 
-  std::vector<Record *> DefaultProperties;
-  for (Record *Rec : IntrProperties)
+  std::vector<const Record *> DefaultProperties;
+  for (const Record *Rec : IntrProperties)
     if (Rec->getValueAsBit("IsDefault"))
       DefaultProperties.push_back(Rec);
 
   std::vector<Record *> Defs = RC.getAllDerivedDefinitions("Intrinsic");
   Intrinsics.reserve(Defs.size());
 
-  for (unsigned I = 0, e = Defs.size(); I != e; ++I)
-    Intrinsics.push_back(CodeGenIntrinsic(Defs[I], DefaultProperties));
+  for (const Record *Def : Defs)
+    Intrinsics.push_back(CodeGenIntrinsic(Def, DefaultProperties));
 
   llvm::sort(Intrinsics,
              [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) {
@@ -54,52 +54,34 @@ CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) {
   Targets.back().Count = Intrinsics.size() - Targets.back().Offset;
 }
 
-CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
-                                   ArrayRef<Record *> DefaultProperties) {
-  TheDef = R;
-  std::string DefName = std::string(R->getName());
+CodeGenIntrinsic::CodeGenIntrinsic(const Record *R,
+                                   ArrayRef<const Record *> DefaultProperties)
+    : TheDef(R) {
+  StringRef DefName = TheDef->getName();
   ArrayRef<SMLoc> DefLoc = R->getLoc();
-  Properties = 0;
-  isOverloaded = false;
-  isCommutative = false;
-  canThrow = false;
-  isNoReturn = false;
-  isNoCallback = false;
-  isNoSync = false;
-  isNoFree = false;
-  isWillReturn = false;
-  isCold = false;
-  isNoDuplicate = false;
-  isNoMerge = false;
-  isConvergent = false;
-  isSpeculatable = false;
-  hasSideEffects = false;
-  isStrictFP = false;
 
-  if (DefName.size() <= 4 || DefName.substr(0, 4) != "int_")
+  if (!DefName.starts_with("int_"))
     PrintFatalError(DefLoc,
                     "Intrinsic '" + DefName + "' does not start with 'int_'!");
 
   EnumName = DefName.substr(4);
 
-  if (R->getValue(
-          "ClangBuiltinName")) // Ignore a missing ClangBuiltinName field.
-    ClangBuiltinName = std::string(R->getValueAsString("ClangBuiltinName"));
-  if (R->getValue("MSBuiltinName")) // Ignore a missing MSBuiltinName field.
-    MSBuiltinName = std::string(R->getValueAsString("MSBuiltinName"));
+  // Ignore a missing ClangBuiltinName field.
+  ClangBuiltinName =
+      R->getValueAsOptionalString("ClangBuiltinName").value_or("");
+  // Ignore a missing MSBuiltinName field.
+  MSBuiltinName = R->getValueAsOptionalString("MSBuiltinName").value_or("");
 
-  TargetPrefix = std::string(R->getValueAsString("TargetPrefix"));
-  Name = std::string(R->getValueAsString("LLVMName"));
+  TargetPrefix = R->getValueAsString("TargetPrefix");
+  Name = R->getValueAsString("LLVMName").str();
 
   if (Name == "") {
     // If an explicit name isn't specified, derive one from the DefName.
-    Name = "llvm.";
-
-    for (unsigned i = 0, e = EnumName.size(); i != e; ++i)
-      Name += (EnumName[i] == '_') ? '.' : EnumName[i];
+    Name = "llvm." + EnumName.str();
+    llvm::replace(Name, '_', '.');
   } else {
     // Verify it starts with "llvm.".
-    if (Name.size() <= 5 || Name.substr(0, 5) != "llvm.")
+    if (!StringRef(Name).starts_with("llvm."))
       PrintFatalError(DefLoc, "Intrinsic '" + DefName +
                                   "'s name does not start with 'llvm.'!");
   }
@@ -107,8 +89,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
   // If TargetPrefix is specified, make sure that Name starts with
   // "llvm.<targetprefix>.".
   if (!TargetPrefix.empty()) {
-    if (Name.size() < 6 + TargetPrefix.size() ||
-        Name.substr(5, 1 + TargetPrefix.size()) != (TargetPrefix + "."))
+    StringRef Prefix = StringRef(Name).drop_front(5); // Drop llvm.
+    if (!Prefix.consume_front(TargetPrefix) || !Prefix.starts_with('.'))
       PrintFatalError(DefLoc, "Intrinsic '" + DefName +
                                   "' does not start with 'llvm." +
                                   TargetPrefix + ".'!");
@@ -129,7 +111,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
   // Parse the intrinsic properties.
   ListInit *PropList = R->getValueAsListInit("IntrProperties");
   for (unsigned i = 0, e = PropList->size(); i != e; ++i) {
-    Record *Property = PropList->getElementAsRecord(i);
+    const Record *Property = PropList->getElementAsRecord(i);
     assert(Property->isSubClassOf("IntrinsicProperty") &&
            "Expected a property!");
 
@@ -137,7 +119,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
   }
 
   // Set default properties to true.
-  setDefaultProperties(R, DefaultProperties);
+  setDefaultProperties(DefaultProperties);
 
   // Also record the SDPatternOperator Properties.
   Properties = parseSDPatternOperatorProperties(R);
@@ -148,16 +130,16 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
 }
 
 void CodeGenIntrinsic::setDefaultProperties(
-    Record *R, ArrayRef<Record *> DefaultProperties) {
+    ArrayRef<const Record *> DefaultProperties) {
   // opt-out of using default attributes.
-  if (R->getValueAsBit("DisableDefaultAttributes"))
+  if (TheDef->getValueAsBit("DisableDefaultAttributes"))
     return;
 
-  for (Record *Rec : DefaultProperties)
+  for (const Record *Rec : DefaultProperties)
     setProperty(Rec);
 }
 
-void CodeGenIntrinsic::setProperty(Record *R) {
+void CodeGenIntrinsic::setProperty(const Record *R) {
   if (R->getName() == "IntrNoMem")
     ME = MemoryEffects::none();
   else if (R->getName() == "IntrReadMem") {
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
index 417063a9490ef19..19e29af2fa85f36 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
@@ -26,12 +26,12 @@ class Record;
 class RecordKeeper;
 
 struct CodeGenIntrinsic {
-  Record *TheDef;       // The actual record defining this intrinsic.
+  const Record *TheDef; // The actual record defining this intrinsic.
   std::string Name;     // The name of the LLVM function "llvm.bswap.i32"
-  std::string EnumName; // The name of the enum "bswap_i32"
-  std::string ClangBuiltinName; // Name of the corresponding GCC builtin, or "".
-  std::string MSBuiltinName;    // Name of the corresponding MS builtin, or "".
-  std::string TargetPrefix;     // Target prefix, e.g. "ppc" for t-s intrinsics.
+  StringRef EnumName;   // The name of the enum "bswap_i32"
+  StringRef ClangBuiltinName; // Name of the corresponding GCC builtin, or "".
+  StringRef MSBuiltinName;    // Name of the corresponding MS builtin, or "".
+  StringRef TargetPrefix;     // Target prefix, e.g. "ppc" for t-s intrinsics.
 
   /// This structure holds the return values and parameter values of an
   /// intrinsic. If the number of return values is > 1, then the intrinsic
@@ -43,13 +43,13 @@ struct CodeGenIntrinsic {
     /// only populated when in the context of a target .td file. When building
     /// Intrinsics.td, this isn't available, because we don't know the target
     /// pointer size.
-    std::vector<Record *> RetTys;
+    std::vector<const Record *> RetTys;
 
     /// The MVT::SimpleValueType for each parameter type. Note that this list is
     /// only populated when in the context of a target .td file.  When building
     /// Intrinsics.td, this isn't available, because we don't know the target
     /// pointer size.
-    std::vector<Record *> ParamTys;
+    std::vector<const Record *> ParamTys;
   };
 
   IntrinsicSignature IS;
@@ -58,54 +58,54 @@ struct CodeGenIntrinsic {
   MemoryEffects ME = MemoryEffects::unknown();
 
   /// SDPatternOperator Properties applied to the intrinsic.
-  unsigned Properties;
+  unsigned Properties = 0;
 
   /// This is set to true if the intrinsic is overloaded by its argument
   /// types.
-  bool isOverloaded;
+  bool isOverloaded = false;
 
   /// True if the intrinsic is commutative.
-  bool isCommutative;
+  bool isCommutative = false;
 
   /// True if the intrinsic can throw.
-  bool canThrow;
+  bool canThrow = false;
 
   /// True if the intrinsic is marked as noduplicate.
-  bool isNoDuplicate;
+  bool isNoDuplicate = false;
 
   /// True if the intrinsic is marked as nomerge.
-  bool isNoMerge;
+  bool isNoMerge = false;
 
   /// True if the intrinsic is no-return.
-  bool isNoReturn;
+  bool isNoReturn = false;
 
   /// True if the intrinsic is no-callback.
-  bool isNoCallback;
+  bool isNoCallback = false;
 
   /// True if the intrinsic is no-sync.
-  bool isNoSync;
+  bool isNoSync = false;
 
   /// True if the intrinsic is no-free.
-  bool isNoFree;
+  bool isNoFree = false;
 
   /// True if the intrinsic is will-return.
-  bool isWillReturn;
+  bool isWillReturn = false;
 
   /// True if the intrinsic is cold.
-  bool isCold;
+  bool isCold = false;
 
   /// True if the intrinsic is marked as convergent.
-  bool isConvergent;
+  bool isConvergent = false;
 
   /// True if the intrinsic has side effects that aren't captured by any
   /// of the other flags.
-  bool hasSideEffects;
+  bool hasSideEffects = false;
 
   // True if the intrinsic is marked as speculatable.
-  bool isSpeculatable;
+  bool isSpeculatable = false;
 
   // True if the intrinsic is marked as strictfp.
-  bool isStrictFP;
+  bool isStrictFP = false;
 
   enum ArgAttrKind {
     NoCapture,
@@ -139,12 +139,12 @@ struct CodeGenIntrinsic {
 
   bool hasProperty(enum SDNP Prop) const { return Properties & (1 << Prop); }
 
-  /// Goes through all IntrProperties that have IsDefault
-  /// value set and sets the property.
-  void setDefaultProperties(Record *R, ArrayRef<Record *> DefaultProperties);
+  /// Goes through all IntrProperties that have IsDefault value set and sets
+  /// the property.
+  void setDefaultProperties(ArrayRef<const Record *> DefaultProperties);
 
-  /// Helper function to set property \p Name to true;
-  void setProperty(Record *R);
+  /// Helper function to set property \p Name to true.
+  void setProperty(const Record *R);
 
   /// Returns true if the parameter at \p ParamIdx is a pointer type. Returns
   /// false if the parameter is not a pointer, or \p ParamIdx is greater than
@@ -155,7 +155,8 @@ struct CodeGenIntrinsic {
 
   bool isParamImmArg(unsigned ParamIdx) const;
 
-  CodeGenIntrinsic(Record *R, ArrayRef<Record *> DefaultProperties);
+  CodeGenIntrinsic(const Record *R,
+                   ArrayRef<const Record *> DefaultProperties = {});
 };
 
 class CodeGenIntrinsicTable {
@@ -163,7 +164,7 @@ class CodeGenIntrinsicTable {
 
 public:
   struct TargetSet {
-    std::string Name;
+    StringRef Name;
     size_t Offset;
     size_t Count;
   };
diff --git a/llvm/utils/TableGen/Basic/SDNodeProperties.cpp b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp
index 2aec41aac6257fb..46babbbc4194463 100644
--- a/llvm/utils/TableGen/Basic/SDNodeProperties.cpp
+++ b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp
@@ -13,9 +13,9 @@
 
 using namespace llvm;
 
-unsigned llvm::parseSDPatternOperatorProperties(Record *R) {
+unsigned llvm::parseSDPatternOperatorProperties(const Record *R) {
   unsigned Properties = 0;
-  for (Record *Property : R->getValueAsListOfDefs("Properties")) {
+  for (const Record *Property : R->getValueAsListOfDefs("Properties")) {
     auto Offset = StringSwitch<unsigned>(Property->getName())
                       .Case("SDNPCommutative", SDNPCommutative)
                       .Case("SDNPAssociative", SDNPAssociative)
diff --git a/llvm/utils/TableGen/Basic/SDNodeProperties.h b/llvm/utils/TableGen/Basic/SDNodeProperties.h
index 10d0bb965fbb6c0..1fe4044edcea236 100644
--- a/llvm/utils/TableGen/Basic/SDNodeProperties.h
+++ b/llvm/utils/TableGen/Basic/SDNodeProperties.h
@@ -32,7 +32,7 @@ enum SDNP {
   SDNPWantParent
 };
 
-unsigned parseSDPatternOperatorProperties(Record *R);
+unsigned parseSDPatternOperatorProperties(const Record *R);
 
 } // namespace llvm
 
diff --git a/llvm/utils/TableGen/Common/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h
index f4ababcd4be68ae..81a5e3e050f2724 100644
--- a/llvm/utils/TableGen/Common/DAGISelMatcher.h
+++ b/llvm/utils/TableGen/Common/DAGISelMatcher.h
@@ -836,7 +836,8 @@ class EmitIntegerMatcher : public Matcher {
 
 public:
   EmitIntegerMatcher(int64_t val, MVT::SimpleValueType vt)
-      : Matcher(EmitInteger), Val(val), VT(vt) {}
+      : Matcher(EmitInteger), Val(SignExtend64(val, MVT(vt).getSizeInBits())),
+        VT(vt) {}
 
   int64_t getValue() const { return Val; }
   MVT::SimpleValueType getVT() const { return VT; }
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 9b9686c627b7522..139bf2d82180323 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -1314,7 +1314,7 @@ void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
   Table << MatchTable::Opcode("GIM_CheckIntrinsicID")
         << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
         << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
-        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName)
+        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName.str())
         << MatchTable::LineBreak;
 }
 
@@ -2103,7 +2103,7 @@ void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table,
                                             RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIR_AddIntrinsicID") << MatchTable::Comment("MI")
         << MatchTable::ULEB128Value(InsnID)
-        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName)
+        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName.str())
         << MatchTable::LineBreak;
 }
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp b/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
index 1d6c4c73a26405d..82fa3c8f3121f05 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
@@ -112,7 +112,7 @@ static const CodeGenIntrinsic *getCodeGenIntrinsic(Record *R) {
 
   auto &Ptr = AllIntrinsics[R];
   if (!Ptr)
-    Ptr = std::make_unique<CodeGenIntrinsic>(R, std::vector<Record *>());
+    Ptr = std::make_unique<CodeGenIntrinsic>(R);
   return Ptr.get();
 }
 
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index 53791618e80fe06..9cc1b5ccb8acb95 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -18,7 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
-#include "llvm/CodeGenTypes/MachineValueType.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/DXILABI.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/TableGen/Error.h"
@@ -54,40 +54,6 @@ struct DXILOperationDesc {
 };
 } // end anonymous namespace
 
-/// Return dxil::ParameterKind corresponding to input LLVMType record
-///
-/// \param R TableGen def record of class LLVMType
-/// \return ParameterKind As defined in llvm/Support/DXILABI.h
-
-static ParameterKind getParameterKind(const Record *R) {
-  auto VTRec = R->getValueAsDef("VT");
-  switch (getValueType(VTRec)) {
-  case MVT::isVoid:
-    return ParameterKind::Void;
-  case MVT::f16:
-    return ParameterKind::Half;
-  case MVT::f32:
-    return ParameterKind::Float;
-  case MVT::f64:
-    return ParameterKind::Double;
-  case MVT::i1:
-    return ParameterKind::I1;
-  case MVT::i8:
-    return ParameterKind::I8;
-  case MVT::i16:
-    return ParameterKind::I16;
-  case MVT::i32:
-    return ParameterKind::I32;
-  case MVT::fAny:
-  case MVT::iAny:
-  case MVT::Any:
-    return ParameterKind::Overload;
-  default:
-    llvm_unreachable(
-        "Support for specified parameter type not yet implemented");
-  }
-}
-
 /// In-place sort TableGen records of class with a field
 ///    Version dxil_version
 /// in the ascending version order.
@@ -134,10 +100,9 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   // llvm/IR/Intrinsics.td
   OverloadParamIndex = -1; // A sigil meaning none.
   for (unsigned i = 0; i < ParamTypeRecsSize; i++) {
-    auto TR = ParamTypeRecs[i];
+    Record *TR = ParamTypeRecs[i];
     // Track operation parameter indices of any overload types
-    auto isAny = TR->getValueAsInt("isAny");
-    if (isAny == 1) {
+    if (TR->getValueAsInt("isOverload")) {
       if (OverloadParamIndex != -1) {
         assert(TR == ParamTypeRecs[OverloadParamIndex] &&
                "Specification of multiple differing overload parameter types "
@@ -148,8 +113,6 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
       if (OverloadParamIndex <= 0)
         OverloadParamIndex = i;
     }
-    if (TR->isAnonymous())
-      PrintFatalError(TR, "Only concrete types are allowed here");
     OpTypes.emplace_back(TR);
   }
 
@@ -208,71 +171,23 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   }
 }
 
-/// Return a string representation of ParameterKind enum
-/// \param Kind Parameter Kind enum value
-/// \return std::string string representation of input Kind
-static std::string getParameterKindStr(ParameterKind Kind) {
-  switch (Kind) {
-  case ParameterKind::Invalid:
-    return "Invalid";
-  case ParameterKind::Void:
-    return "Void";
-  case ParameterKind::Half:
-    return "Half";
-  case ParameterKind::Float:
-    return "Float";
-  case ParameterKind::Double:
-    return "Double";
-  case ParameterKind::I1:
-    return "I1";
-  case ParameterKind::I8:
-    return "I8";
-  case ParameterKind::I16:
-    return "I16";
-  case ParameterKind::I32:
-    return "I32";
-  case ParameterKind::I64:
-    return "I64";
-  case ParameterKind::Overload:
-    return "Overload";
-  case ParameterKind::CBufferRet:
-    return "CBufferRet";
-  case ParameterKind::ResourceRet:
-    return "ResourceRet";
-  case ParameterKind::DXILHandle:
-    return "DXILHandle";
-  }
-  llvm_unreachable("Unknown llvm::dxil::ParameterKind enum");
-}
-
 /// Return a string representation of OverloadKind enum that maps to
 /// input LLVMType record
 /// \param R TableGen def record of class LLVMType
 /// \return std::string string representation of OverloadKind
 
-static std::string getOverloadKindStr(const Record *R) {
-  Record *VTRec = R->getValueAsDef("VT");
-  switch (getValueType(VTRec)) {
-  case MVT::f16:
-    return "OverloadKind::HALF";
-  case MVT::f32:
-    return "OverloadKind::FLOAT";
-  case MVT::f64:
-    return "OverloadKind::DOUBLE";
-  case MVT::i1:
-    return "OverloadKind::I1";
-  case MVT::i8:
-    return "OverloadKind::I8";
-  case MVT::i16:
-    return "OverloadKind::I16";
-  case MVT::i32:
-    return "OverloadKind::I32";
-  case MVT::i64:
-    return "OverloadKind::I64";
-  default:
-    llvm_unreachable("Support for specified fixed type option for overload "
-                     "type not supported");
-  }
+static StringRef getOverloadKindStr(const Record *R) {
+  // TODO: This is a hack. We need to rework how we're handling the set of
+  // overloads to avoid this business with the separate OverloadKind enum.
+  return StringSwitch<StringRef>(R->getName())
+      .Case("HalfTy", "OverloadKind::HALF")
+      .Case("FloatTy", "OverloadKind::FLOAT")
+      .Case("DoubleTy", "OverloadKind::DOUBLE")
+      .Case("Int1Ty", "OverloadKind::I1")
+      .Case("Int8Ty", "OverloadKind::I8")
+      .Case("Int16Ty", "OverloadKind::I16")
+      .Case("Int32Ty", "OverloadKind::I32")
+      .Case("Int64Ty", "OverloadKind::I64");
 }
 
 /// Return a string representation of valid overload information denoted
@@ -417,8 +332,7 @@ static void emitDXILOpCodes(std::vector<DXILOperationDesc> &Ops,
 }
 
 /// Emit a list of DXIL op classes
-static void emitDXILOpClasses(RecordKeeper &Records,
-                              raw_ostream &OS) {
+static void emitDXILOpClasses(RecordKeeper &Records, raw_ostream &OS) {
   OS << "#ifdef DXIL_OPCLASS\n";
   std::vector<Record *> OpClasses =
       Records.getAllDerivedDefinitions("DXILOpClass");
@@ -428,6 +342,35 @@ static void emitDXILOpClasses(RecordKeeper &Records,
   OS << "#endif\n\n";
 }
 
+/// Emit a list of DXIL op parameter types
+static void emitDXILOpParamTypes(RecordKeeper &Records, raw_ostream &OS) {
+  OS << "#ifdef DXIL_OP_PARAM_TYPE\n";
+  std::vector<Record *> OpClasses =
+      Records.getAllDerivedDefinitions("DXILOpParamType");
+  for (Record *OpClass : OpClasses)
+    OS << "DXIL_OP_PARAM_TYPE(" << OpClass->getName() << ")\n";
+  OS << "#undef DXIL_OP_PARAM_TYPE\n";
+  OS << "#endif\n\n";
+}
+
+/// Emit a list of DXIL op function types
+static void emitDXILOpFunctionTypes(ArrayRef<DXILOperationDesc> Ops,
+                                    raw_ostream &OS) {
+  OS << "#ifndef DXIL_OP_FUNCTION_TYPE\n";
+  OS << "#define DXIL_OP_FUNCTION_TYPE(OpCode, RetType, ...)\n";
+  OS << "#endif\n";
+  for (const DXILOperationDesc &Op : Ops) {
+    OS << "DXIL_OP_FUNCTION_TYPE(dxil::OpCode::" << Op.OpName;
+    for (const Record *Rec : Op.OpTypes)
+      OS << ", dxil::OpParamType::" << Rec->getName();
+    // If there are no arguments, we need an empty comma for the varargs
+    if (Op.OpTypes.size() == 1)
+      OS << ", ";
+    OS << ")\n";
+  }
+  OS << "#undef DXIL_OP_FUNCTION_TYPE\n";
+}
+
 /// Emit map of DXIL operation to LLVM or DirectX intrinsic
 /// \param A vector of DXIL Ops
 /// \param Output stream
@@ -454,9 +397,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
   // Collect Names.
   SequenceToOffsetTable<std::string> OpClassStrings;
   SequenceToOffsetTable<std::string> OpStrings;
-  SequenceToOffsetTable<SmallVector<ParameterKind>> Parameters;
 
-  StringMap<SmallVector<ParameterKind>> ParameterMap;
   StringSet<> ClassSet;
   for (auto &Op : Ops) {
     OpStrings.add(Op.OpName);
@@ -465,18 +406,11 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
       continue;
     ClassSet.insert(Op.OpClass);
     OpClassStrings.add(Op.OpClass.data());
-    SmallVector<ParameterKind> ParamKindVec;
-    for (unsigned i = 0; i < Op.OpTypes.size(); i++) {
-      ParamKindVec.emplace_back(getParameterKind(Op.OpTypes[i]));
-    }
-    ParameterMap[Op.OpClass] = ParamKindVec;
-    Parameters.add(ParamKindVec);
   }
 
   // Layout names.
   OpStrings.layout();
   OpClassStrings.layout();
-  Parameters.layout();
 
   // Emit access function getOpcodeProperty() that embeds DXIL Operation table
   // with entries of type struct OpcodeProperty.
@@ -492,8 +426,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
        << getOverloadMaskString(Op.OverloadRecs) << ", "
        << getStageMaskString(Op.StageRecs) << ", "
        << getAttributeMaskString(Op.AttrRecs) << ", " << Op.OverloadParamIndex
-       << ", " << Op.OpTypes.size() << ", "
-       << Parameters.get(ParameterMap[Op.OpClass]) << " }";
+       << " }";
     Prefix = ",\n";
   }
   OS << "  };\n";
@@ -531,21 +464,6 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
 
   OS << "  unsigned Index = Prop.OpCodeClassNameOffset;\n";
   OS << "  return DXILOpCodeClassNameTable + Index;\n";
-  OS << "}\n ";
-
-  OS << "static const ParameterKind *getOpCodeParameterKind(const "
-        "OpCodeProperty &Prop) "
-        "{\n\n";
-  OS << "  static const ParameterKind DXILOpParameterKindTable[] = {\n";
-  Parameters.emit(
-      OS,
-      [](raw_ostream &ParamOS, ParameterKind Kind) {
-        ParamOS << "ParameterKind::" << getParameterKindStr(Kind);
-      },
-      "ParameterKind::Invalid");
-  OS << "  };\n\n";
-  OS << "  unsigned Index = Prop.ParameterTableOffset;\n";
-  OS << "  return DXILOpParameterKindTable + Index;\n";
   OS << "}\n\n";
 }
 
@@ -611,6 +529,8 @@ static void EmitDXILOperation(RecordKeeper &Records, raw_ostream &OS) {
 
   emitDXILOpCodes(DXILOps, OS);
   emitDXILOpClasses(Records, OS);
+  emitDXILOpParamTypes(Records, OS);
+  emitDXILOpFunctionTypes(DXILOps, OS);
   emitDXILIntrinsicMap(DXILOps, OS);
   OS << "#ifdef DXIL_OP_OPERATION_TABLE\n\n";
   emitDXILOperationTableDataStructs(Records, OS);
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index a7e99fa4c05068c..b1307153e9109bc 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -35,18 +35,18 @@
 #include <vector>
 using namespace llvm;
 
-cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
-cl::opt<std::string>
+static cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
+static cl::opt<std::string>
     IntrinsicPrefix("intrinsic-prefix",
                     cl::desc("Generate intrinsics with this target prefix"),
                     cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat));
 
 namespace {
 class IntrinsicEmitter {
-  RecordKeeper &Records;
+  const RecordKeeper &Records;
 
 public:
-  IntrinsicEmitter(RecordKeeper &R) : Records(R) {}
+  IntrinsicEmitter(const RecordKeeper &R) : Records(R) {}
 
   void run(raw_ostream &OS, bool Enums);
 
@@ -123,7 +123,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
     std::vector<std::string> KnownTargets;
     for (const auto &Target : Ints.Targets)
       if (!Target.Name.empty())
-        KnownTargets.push_back(Target.Name);
+        KnownTargets.push_back(Target.Name.str());
     PrintFatalError("tried to generate intrinsics for unknown target " +
                     IntrinsicPrefix +
                     "\nKnown targets are: " + join(KnownTargets, ", ") + "\n");
@@ -136,12 +136,11 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
     std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
     OS << "#ifndef LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n";
     OS << "#define LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n\n";
-    OS << "namespace llvm {\n";
-    OS << "namespace Intrinsic {\n";
+    OS << "namespace llvm::Intrinsic {\n";
     OS << "enum " << UpperPrefix << "Intrinsics : unsigned {\n";
   }
 
-  OS << "// Enum values for intrinsics\n";
+  OS << "// Enum values for intrinsics.\n";
   for (unsigned i = Set->Offset, e = Set->Offset + Set->Count; i != e; ++i) {
     OS << "    " << Ints[i].EnumName;
 
@@ -162,8 +161,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
     OS << "#endif\n\n";
   } else {
     OS << "}; // enum\n";
-    OS << "} // namespace Intrinsic\n";
-    OS << "} // namespace llvm\n\n";
+    OS << "} // namespace llvm::Intrinsic\n\n";
     OS << "#endif\n";
   }
 }
@@ -171,7 +169,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
 void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) {
   if (!IntrinsicPrefix.empty())
     return;
-  OS << "// llvm::Intrinsic::IITDescriptor::ArgKind\n";
+  OS << "// llvm::Intrinsic::IITDescriptor::ArgKind.\n";
   OS << "#ifdef GET_INTRINSIC_ARGKIND\n";
   if (auto RecArgKind = Records.getDef("ArgKind")) {
     for (auto &RV : RecArgKind->getValues())
@@ -205,7 +203,7 @@ void IntrinsicEmitter::EmitIITInfo(raw_ostream &OS) {
 
 void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
                                       raw_ostream &OS) {
-  OS << "// Target mapping\n";
+  OS << "// Target mapping.\n";
   OS << "#ifdef GET_INTRINSIC_TARGET_DATA\n";
   OS << "struct IntrinsicTargetInfo {\n"
      << "  llvm::StringLiteral Name;\n"
@@ -222,19 +220,19 @@ void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
 
 void IntrinsicEmitter::EmitIntrinsicToNameTable(
     const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
-  OS << "// Intrinsic ID to name table\n";
+  OS << "// Intrinsic ID to name table.\n";
   OS << "#ifdef GET_INTRINSIC_NAME_TABLE\n";
   OS << "  // Note that entry #0 is the invalid intrinsic!\n";
-  for (unsigned i = 0, e = Ints.size(); i != e; ++i)
-    OS << "  \"" << Ints[i].Name << "\",\n";
+  for (const auto &Int : Ints)
+    OS << "  \"" << Int.Name << "\",\n";
   OS << "#endif\n\n";
 }
 
 void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
     const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
-  OS << "// Intrinsic ID to overload bitset\n";
+  OS << "// Intrinsic ID to overload bitset.\n";
   OS << "#ifdef GET_INTRINSIC_OVERLOAD_TABLE\n";
-  OS << "static const uint8_t OTable[] = {\n";
+  OS << "static constexpr uint8_t OTable[] = {\n";
   OS << "  0";
   for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
     // Add one to the index so we emit a null bit for the invalid #0 intrinsic.
@@ -315,7 +313,7 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
   OS << "// Global intrinsic function declaration type table.\n";
   OS << "#ifdef GET_INTRINSIC_GENERATOR_GLOBAL\n";
 
-  OS << "static const unsigned IIT_Table[] = {\n  ";
+  OS << "static constexpr unsigned IIT_Table[] = {\n  ";
 
   for (unsigned i = 0, e = FixedEncodings.size(); i != e; ++i) {
     if ((i & 7) == 7)
@@ -338,7 +336,7 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
   OS << "0\n};\n\n";
 
   // Emit the shared table of register lists.
-  OS << "static const unsigned char IIT_LongEncodingTable[] = {\n";
+  OS << "static constexpr unsigned char IIT_LongEncodingTable[] = {\n";
   if (!LongEncodingTable.empty())
     LongEncodingTable.emit(OS, printIITEntry);
   OS << "  255\n};\n\n";
@@ -346,72 +344,44 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
   OS << "#endif\n\n"; // End of GET_INTRINSIC_GENERATOR_GLOBAL
 }
 
-namespace {
-std::optional<bool> compareFnAttributes(const CodeGenIntrinsic *L,
-                                        const CodeGenIntrinsic *R) {
-  // Sort throwing intrinsics after non-throwing intrinsics.
-  if (L->canThrow != R->canThrow)
-    return R->canThrow;
-
-  if (L->isNoDuplicate != R->isNoDuplicate)
-    return R->isNoDuplicate;
-
-  if (L->isNoMerge != R->isNoMerge)
-    return R->isNoMerge;
-
-  if (L->isNoReturn != R->isNoReturn)
-    return R->isNoReturn;
-
-  if (L->isNoCallback != R->isNoCallback)
-    return R->isNoCallback;
-
-  if (L->isNoSync != R->isNoSync)
-    return R->isNoSync;
-
-  if (L->isNoFree != R->isNoFree)
-    return R->isNoFree;
-
-  if (L->isWillReturn != R->isWillReturn)
-    return R->isWillReturn;
-
-  if (L->isCold != R->isCold)
-    return R->isCold;
+static bool compareFnAttributes(const CodeGenIntrinsic *L,
+                                const CodeGenIntrinsic *R, bool Default) {
+  auto TieBoolAttributes = [](const CodeGenIntrinsic *I) -> auto {
+    // Sort throwing intrinsics after non-throwing intrinsics.
+    return std::tie(I->canThrow, I->isNoDuplicate, I->isNoMerge, I->isNoReturn,
+                    I->isNoCallback, I->isNoSync, I->isNoFree, I->isWillReturn,
+                    I->isCold, I->isConvergent, I->isSpeculatable,
+                    I->hasSideEffects, I->isStrictFP);
+  };
 
-  if (L->isConvergent != R->isConvergent)
-    return R->isConvergent;
+  auto TieL = TieBoolAttributes(L);
+  auto TieR = TieBoolAttributes(R);
 
-  if (L->isSpeculatable != R->isSpeculatable)
-    return R->isSpeculatable;
-
-  if (L->hasSideEffects != R->hasSideEffects)
-    return R->hasSideEffects;
-
-  if (L->isStrictFP != R->isStrictFP)
-    return R->isStrictFP;
+  if (TieL != TieR)
+    return TieL < TieR;
 
   // Try to order by readonly/readnone attribute.
   uint32_t LK = L->ME.toIntValue();
   uint32_t RK = R->ME.toIntValue();
   if (LK != RK)
-    return (LK > RK);
+    return LK > RK;
 
-  return std::nullopt;
+  return Default;
 }
 
+namespace {
 struct FnAttributeComparator {
   bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
-    return compareFnAttributes(L, R).value_or(false);
+    return compareFnAttributes(L, R, false);
   }
 };
 
 struct AttributeComparator {
   bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
-    if (std::optional<bool> Res = compareFnAttributes(L, R))
-      return *Res;
-
-    // Order by argument attributes.
+    // Order by argument attributes if function attributes are equal.
     // This is reliable because each side is already sorted internally.
-    return (L->ArgumentAttributes < R->ArgumentAttributes);
+    return compareFnAttributes(L, R,
+                               L->ArgumentAttributes < R->ArgumentAttributes);
   }
 };
 } // End anonymous namespace
@@ -559,7 +529,7 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
 
   // Emit an array of AttributeList.  Most intrinsics will have at least one
   // entry, for the function itself (index ~1), which is usually nounwind.
-  OS << "  static const uint16_t IntrinsicsToAttributesMap[] = {\n";
+  OS << "  static constexpr uint16_t IntrinsicsToAttributesMap[] = {\n";
 
   for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
     const CodeGenIntrinsic &intrinsic = Ints[i];
@@ -624,31 +594,34 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
 
 void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     const CodeGenIntrinsicTable &Ints, bool IsClang, raw_ostream &OS) {
-  StringRef CompilerName = (IsClang ? "Clang" : "MS");
-  StringRef UpperCompilerName = (IsClang ? "CLANG" : "MS");
-  typedef std::map<std::string, std::map<std::string, std::string>> BIMTy;
+  StringRef CompilerName = IsClang ? "Clang" : "MS";
+  StringRef UpperCompilerName = IsClang ? "CLANG" : "MS";
+  // map<TargetPrefix, map<BuiltinName, EnumName>>. Note that we iterate over
+  // both maps in the code below. For the inner map, entries need to be emitted
+  // in the sorted order of `BuiltinName` because we use std::lower_bound to
+  // search these entries. For the outer map, it doesn't need be be sorted, but
+  // we use a map to eliminate non-determinism in the emitted code.
+  typedef std::map<StringRef, std::map<StringRef, StringRef>> BIMTy;
   BIMTy BuiltinMap;
   StringToOffsetTable Table;
-  for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
-    const std::string &BuiltinName =
-        IsClang ? Ints[i].ClangBuiltinName : Ints[i].MSBuiltinName;
-    if (!BuiltinName.empty()) {
-      // Get the map for this target prefix.
-      std::map<std::string, std::string> &BIM =
-          BuiltinMap[Ints[i].TargetPrefix];
-
-      if (!BIM.insert(std::pair(BuiltinName, Ints[i].EnumName)).second)
-        PrintFatalError(Ints[i].TheDef->getLoc(),
-                        "Intrinsic '" + Ints[i].TheDef->getName() +
-                            "': duplicate " + CompilerName + " builtin name!");
-      Table.GetOrAddStringOffset(BuiltinName);
-    }
+  for (const CodeGenIntrinsic &Int : Ints) {
+    StringRef BuiltinName = IsClang ? Int.ClangBuiltinName : Int.MSBuiltinName;
+    if (BuiltinName.empty())
+      continue;
+    // Get the map for this target prefix.
+    std::map<StringRef, StringRef> &BIM = BuiltinMap[Int.TargetPrefix];
+
+    if (!BIM.insert(std::pair(BuiltinName, Int.EnumName)).second)
+      PrintFatalError(Int.TheDef->getLoc(),
+                      "Intrinsic '" + Int.TheDef->getName() + "': duplicate " +
+                          CompilerName + " builtin name!");
+    Table.GetOrAddStringOffset(BuiltinName);
   }
 
   OS << "// Get the LLVM intrinsic that corresponds to a builtin.\n";
   OS << "// This is used by the C front-end.  The builtin name is passed\n";
   OS << "// in as BuiltinName, and a target prefix (e.g. 'ppc') is passed\n";
-  OS << "// in as TargetPrefix.  The result is assigned to 'IntrinsicID'.\n";
+  OS << "// in as TargetPrefix. The result is assigned to 'IntrinsicID'.\n";
   OS << "#ifdef GET_LLVM_INTRINSIC_FOR_" << UpperCompilerName << "_BUILTIN\n";
 
   OS << "Intrinsic::ID Intrinsic::getIntrinsicFor" << CompilerName
@@ -662,7 +635,7 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     return;
   }
 
-  OS << "  static const char BuiltinNames[] = {\n";
+  OS << "  static constexpr char BuiltinNames[] = {\n";
   Table.EmitCharArray(OS);
   OS << "  };\n\n";
 
@@ -689,7 +662,7 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     OS << "{\n";
 
     // Emit the comparisons for this target prefix.
-    OS << "    static const BuiltinEntry " << I.first << "Names[] = {\n";
+    OS << "    static constexpr BuiltinEntry " << I.first << "Names[] = {\n";
     for (const auto &P : I.second) {
       OS << "      {Intrinsic::" << P.second << ", "
          << Table.GetOrAddStringOffset(P.first) << "}, // " << P.first << "\n";
@@ -703,8 +676,7 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     OS << "      return I->IntrinID;\n";
     OS << "  }\n";
   }
-  OS << "  return ";
-  OS << "Intrinsic::not_intrinsic;\n";
+  OS << "  return Intrinsic::not_intrinsic;\n";
   OS << "}\n";
   OS << "#endif\n\n";
 }
@@ -721,4 +693,4 @@ static void EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS) {
 }
 
 static TableGen::Emitter::Opt Y("gen-intrinsic-impl", EmitIntrinsicImpl,
-                                "Generate intrinsic information");
+                                "Generate intrinsic implementation code");
diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp
index ac50c9ed2aeba96..59ae3bf3daedcb8 100644
--- a/llvm/utils/TableGen/SearchableTableEmitter.cpp
+++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp
@@ -125,7 +125,7 @@ class SearchableTableEmitter {
     else if (BitInit *BI = dyn_cast<BitInit>(I))
       return BI->getValue() ? "true" : "false";
     else if (Field.IsIntrinsic)
-      return "Intrinsic::" + getIntrinsic(I).EnumName;
+      return "Intrinsic::" + getIntrinsic(I).EnumName.str();
     else if (Field.IsInstruction)
       return I->getAsString();
     else if (Field.Enum) {
@@ -148,8 +148,7 @@ class SearchableTableEmitter {
   CodeGenIntrinsic &getIntrinsic(Init *I) {
     std::unique_ptr<CodeGenIntrinsic> &Intr = Intrinsics[I];
     if (!Intr)
-      Intr = std::make_unique<CodeGenIntrinsic>(cast<DefInit>(I)->getDef(),
-                                                std::vector<Record *>());
+      Intr = std::make_unique<CodeGenIntrinsic>(cast<DefInit>(I)->getDef());
     return *Intr;
   }
 
diff --git a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
index 1708af8612bc287..1dd34f2a077bc52 100644
--- a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
@@ -3,7 +3,7 @@ import("//clang/utils/TableGen/clang_tablegen.gni")
 clang_tablegen("Opcodes") {
   visibility = [ ":AST" ]
   args = [ "-gen-clang-opcodes" ]
-  td_file = "Interp/Opcodes.td"
+  td_file = "ByteCode/Opcodes.td"
 }
 
 clang_tablegen("AttrDocTable") {
@@ -92,31 +92,31 @@ static_library("AST") {
     "ExternalASTSource.cpp",
     "FormatString.cpp",
     "InheritViz.cpp",
-    "Interp/ByteCodeEmitter.cpp",
-    "Interp/Compiler.cpp",
-    "Interp/Context.cpp",
-    "Interp/Descriptor.cpp",
-    "Interp/Disasm.cpp",
-    "Interp/DynamicAllocator.cpp",
-    "Interp/EvalEmitter.cpp",
-    "Interp/EvaluationResult.cpp",
-    "Interp/Floating.cpp",
-    "Interp/Frame.cpp",
-    "Interp/Function.cpp",
-    "Interp/Interp.cpp",
-    "Interp/InterpBlock.cpp",
-    "Interp/InterpBuiltin.cpp",
-    "Interp/InterpFrame.cpp",
-    "Interp/InterpShared.cpp",
-    "Interp/InterpStack.cpp",
-    "Interp/InterpState.cpp",
-    "Interp/MemberPointer.cpp",
-    "Interp/Pointer.cpp",
-    "Interp/PrimType.cpp",
-    "Interp/Program.cpp",
-    "Interp/Record.cpp",
-    "Interp/Source.cpp",
-    "Interp/State.cpp",
+    "ByteCode/ByteCodeEmitter.cpp",
+    "ByteCode/Compiler.cpp",
+    "ByteCode/Context.cpp",
+    "ByteCode/Descriptor.cpp",
+    "ByteCode/Disasm.cpp",
+    "ByteCode/DynamicAllocator.cpp",
+    "ByteCode/EvalEmitter.cpp",
+    "ByteCode/EvaluationResult.cpp",
+    "ByteCode/Floating.cpp",
+    "ByteCode/Frame.cpp",
+    "ByteCode/Function.cpp",
+    "ByteCode/Interp.cpp",
+    "ByteCode/InterpBlock.cpp",
+    "ByteCode/InterpBuiltin.cpp",
+    "ByteCode/InterpFrame.cpp",
+    "ByteCode/InterpShared.cpp",
+    "ByteCode/InterpStack.cpp",
+    "ByteCode/InterpState.cpp",
+    "ByteCode/MemberPointer.cpp",
+    "ByteCode/Pointer.cpp",
+    "ByteCode/PrimType.cpp",
+    "ByteCode/Program.cpp",
+    "ByteCode/Record.cpp",
+    "ByteCode/Source.cpp",
+    "ByteCode/State.cpp",
     "ItaniumCXXABI.cpp",
     "ItaniumMangle.cpp",
     "JSONNodeDumper.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn
index 2ea3cdd393bda50..586b33f5df9daf7 100644
--- a/llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn
@@ -7,9 +7,7 @@ static_library("Rewrite") {
     "//llvm/lib/Support",
   ]
   sources = [
-    "DeltaTree.cpp",
     "HTMLRewrite.cpp",
-    "RewriteRope.cpp",
     "Rewriter.cpp",
     "TokenRewriter.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/clang/unittests/AST/Interp/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/AST/ByteCode/BUILD.gn
similarity index 100%
rename from llvm/utils/gn/secondary/clang/unittests/AST/Interp/BUILD.gn
rename to llvm/utils/gn/secondary/clang/unittests/AST/ByteCode/BUILD.gn
diff --git a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
index 8b98e3beec284d5..a6a4a5708341f0d 100644
--- a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
@@ -3,7 +3,7 @@ import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 group("unittests") {
   deps = [
     "AST:ASTTests",
-    "AST/Interp:InterpTests",
+    "AST/ByteCode:InterpTests",
     "ASTMatchers:ASTMatchersTests",
     "ASTMatchers/Dynamic:DynamicASTMatchersTests",
     "Basic:BasicTests",
diff --git a/llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn
index 52003f8a8db0fa4..88eeff60d05d406 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn
@@ -9,8 +9,5 @@ unittest("RewriteTests") {
     "//clang/lib/Tooling",
     "//llvm/lib/Support",
   ]
-  sources = [
-    "RewriteBufferTest.cpp",
-    "RewriterTest.cpp",
-  ]
+  sources = [ "RewriterTest.cpp" ]
 }
diff --git a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
index 07b0a7ac66823ca..5faa365bb7bdbcd 100644
--- a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
@@ -35,6 +35,7 @@ static_library("Utility") {
     "Log.cpp",
     "NameMatches.cpp",
     "ProcessInfo.cpp",
+    "RealpathPrefixes.cpp",
     "RegisterValue.cpp",
     "RegularExpression.cpp",
     "Scalar.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
index 31f567d6df159d4..b5cd80034ef05e1 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -39,7 +39,9 @@ static_library("Orc") {
     "LLJIT.cpp",
     "Layer.cpp",
     "LazyReexports.cpp",
+    "LoadRelocatableObject.cpp",
     "LookupAndRecordAddrs.cpp",
+    "MachO.cpp",
     "MachOPlatform.cpp",
     "Mangling.cpp",
     "MapperJITLinkMemoryManager.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index d6b6b71a1e59b3b..d10a6e4be3770c1 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -73,6 +73,7 @@ static_library("Support") {
     "Debug.cpp",
     "DebugCounter.cpp",
     "DeltaAlgorithm.cpp",
+    "DeltaTree.cpp",
     "DivisionByConstantInfo.cpp",
     "DynamicAPInt.cpp",
     "ELFAttributeParser.cpp",
@@ -123,6 +124,8 @@ static_library("Support") {
     "RWMutex.cpp",
     "RandomNumberGenerator.cpp",
     "Regex.cpp",
+    "RewriteBuffer.cpp",
+    "RewriteRope.cpp",
     "SHA1.cpp",
     "SHA256.cpp",
     "ScaledNumber.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 9fb898445ed6fa6..c9b1b4d9ac31359 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -139,7 +139,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUAtomicOptimizer.cpp",
     "AMDGPUAttributor.cpp",
     "AMDGPUCallLowering.cpp",
-    "AMDGPUCodeGenPassBuilder.cpp",
     "AMDGPUCodeGenPrepare.cpp",
     "AMDGPUCombinerHelper.cpp",
     "AMDGPUCtorDtorLowering.cpp",
@@ -209,7 +208,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "GCNVOPDUtils.cpp",
     "R600AsmPrinter.cpp",
     "R600ClauseMergePass.cpp",
-    "R600CodeGenPassBuilder.cpp",
     "R600ControlFlowFinalizer.cpp",
     "R600EmitClauseMarkers.cpp",
     "R600ExpandSpecialInstrs.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
index f080c06f10dfe11..e297aa9da4e50d6 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
@@ -55,6 +55,7 @@ static_library("Scalar") {
     "LoopSimplifyCFG.cpp",
     "LoopSink.cpp",
     "LoopStrengthReduce.cpp",
+    "LoopTermFold.cpp",
     "LoopUnrollAndJamPass.cpp",
     "LoopUnrollPass.cpp",
     "LoopVersioningLICM.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
index c081bc03c66957e..c27faaaecf30d16 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
@@ -71,6 +71,7 @@ unittest("ADTTests") {
     "PostOrderIteratorTest.cpp",
     "PriorityWorklistTest.cpp",
     "RangeAdapterTest.cpp",
+    "RewriteBufferTest.cpp",
     "SCCIteratorTest.cpp",
     "STLExtrasTest.cpp",
     "STLForwardCompatTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
index 4749ef0a733da34..ee635fd145c035c 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
@@ -28,6 +28,7 @@ unittest("AnalysisTests") {
     "DomTreeUpdaterTest.cpp",
     "FunctionPropertiesAnalysisTest.cpp",
     "GlobalsModRefTest.cpp",
+    "GraphWriterTest.cpp",
     "IRSimilarityIdentifierTest.cpp",
     "IVDescriptorsTest.cpp",
     "InlineCostTest.cpp",
diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py
index 8037094a910678a..6c29cca891581fa 100644
--- a/llvm/utils/lit/lit/formats/googletest.py
+++ b/llvm/utils/lit/lit/formats/googletest.py
@@ -334,7 +334,11 @@ def remove_gtest(tests):
                             returnCode = lit.Test.SKIPPED
                         elif "failures" in testinfo:
                             has_failure_in_shard = True
-                            returnCode = lit.Test.FAIL
+                            returnCode = (
+                                lit.Test.XFAIL
+                                if test.isExpectedToFail()
+                                else lit.Test.FAIL
+                            )
                             output = header
                             for fail in testinfo["failures"]:
                                 output += fail["failure"] + "\n"
diff --git a/mlir/docs/BytecodeFormat.md b/mlir/docs/BytecodeFormat.md
index 869b2d8f0a00b66..ebc94c9f0d8ba6f 100644
--- a/mlir/docs/BytecodeFormat.md
+++ b/mlir/docs/BytecodeFormat.md
@@ -1,10 +1,10 @@
 # MLIR Bytecode Format
 
-This documents describes the MLIR bytecode format and its encoding.
+This document describes the MLIR bytecode format and its encoding.
 This format is versioned and stable: we don't plan to ever break
-compatibility, that is a dialect should be able to deserialize and
-older bytecode. Similarly, we support back-deployment we an older
-version of the format can be targetted.
+compatibility, that is a dialect should be able to deserialize any
+older bytecode. Similarly, we support back-deployment so that an
+older version of the format can be targetted.
 
 That said, it is important to realize that the promises of the
 bytecode format are made assuming immutable dialects: the format
@@ -19,7 +19,7 @@ information while decoding the input IR, and gives an opportunity
 to each dialect for which a version is present to perform IR
 upgrades post-parsing through the `upgradeFromVersion` method.
 There is no restriction on what kind of information a dialect
-is allowed to encode to model its versioning
+is allowed to encode to model its versioning.
 
 [TOC]
 
@@ -172,16 +172,13 @@ dialects that were also referenced.
 ```
 dialect_section {
   numDialects: varint,
-  dialectNames: varint[],
-  numTotalOpNames: varint,
-  opNames: op_name_group[]
+  dialectNames: dialect_name_group[],
+  opNames: dialect_ops_group[]  // ops grouped by dialect
 }
 
-op_name_group {
-  dialect: varint // (dialectID << 1) | (hasVersion),
-  version : dialect_version_section
-  numOpNames: varint,
-  opNames: varint[]
+dialect_name_group {
+  nameAndIsVersioned: varint  // (dialectID << 1) | (hasVersion),
+  version: dialect_version_section  // only if versioned
 }
 
 dialect_version_section {
@@ -189,6 +186,15 @@ dialect_version_section {
   version: byte[]
 }
 
+dialect_ops_group {
+  dialect: varint,
+  numOpNames: varint,
+  opNames: op_name_group[]
+}
+
+op_name_group {
+  nameAndIsRegistered: varint  // (nameID << 1) | (isRegisteredOp)
+}
 ```
 
 Dialects are encoded as a `varint` containing the index to the name string
@@ -196,7 +202,7 @@ within the string section, plus a flag indicating whether the dialect is
 versioned. Operation names are encoded in groups by dialect, with each group
 containing the dialect, the number of operation names, and the array of indexes
 to each name within the string section. The version is encoded as a nested
-section.
+section for each dialect.
 
 ### Attribute/Type Sections
 
@@ -249,9 +255,9 @@ its assembly format, or via a custom dialect defined encoding.
 
 In the case where a dialect does not define a method for encoding the attribute
 or type, the textual assembly format of that attribute or type is used as a
-fallback. For example, a type of `!bytecode.type` would be encoded as the null
-terminated string "!bytecode.type". This ensures that every attribute and type
-may be encoded, even if the owning dialect has not yet opted in to a more
+fallback. For example, a type `!bytecode.type<42>` would be encoded as the null
+terminated string "!bytecode.type<42>". This ensures that every attribute and
+type can be encoded, even if the owning dialect has not yet opted in to a more
 efficient serialization.
 
 TODO: We shouldn't redundantly encode the dialect name here, we should use a
@@ -259,9 +265,9 @@ reference to the parent dialect instead.
 
 ##### Dialect Defined Encoding
 
-In addition to the assembly format fallback, dialects may also provide a custom
-encoding for their attributes and types. Custom encodings are very beneficial in
-that they are significantly smaller and faster to read and write.
+As an alternative to the assembly format fallback, dialects may also provide a
+custom encoding for their attributes and types. Custom encodings are very
+beneficial in that they are significantly smaller and faster to read and write.
 
 Dialects can opt-in to providing custom encodings by implementing the
 `BytecodeDialectInterface`. This interface provides hooks, namely
@@ -377,7 +383,7 @@ uselist {
 
 The encoding of an operation is important because this is generally the most
 commonly appearing structure in the bytecode. A single encoding is used for
-every type of operation. Given this prevelance, many of the fields of an
+every type of operation. Given this prevalence, many of the fields of an
 operation are optional. The `encodingMask` field is a bitmask which indicates
 which of the components of the operation are present.
 
diff --git a/mlir/docs/Dialects/ArmSME.md b/mlir/docs/Dialects/ArmSME.md
index ce0a76ed60f19cf..d38a3e581f48d13 100644
--- a/mlir/docs/Dialects/ArmSME.md
+++ b/mlir/docs/Dialects/ArmSME.md
@@ -12,14 +12,14 @@ operation to Arm SME
 integration tests for reference:
 
 * [Linalg/CPU/ArmSME/matmul.mlir](https://github.com/llvm/llvm-project/blob/main/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir)
-* [Vector/CPU/ArmSME/test-outerproduct-f64.mlir](https://github.com/llvm/llvm-project/blob/main/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-outerproduct-f64.mlir)
+* [Vector/CPU/ArmSME/outerproduct-f64.mlir](https://github.com/llvm/llvm-project/blob/main/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir)
 
 In order to run ArmSME integration tests, include these flags in the CMake
 invocation when configuring LLVM and MLIR:
 ```bash
   -DMLIR_INCLUDE_INTEGRATION_TESTS=On
   -DMLIR_RUN_ARM_SME_TESTS=On
-  -DARM_EMULATOR_EXECUTABLE=<path-to-emulator> 
+  -DARM_EMULATOR_EXECUTABLE=<path-to-emulator>
 ```
 
 These tests are run "post-commit" by the
diff --git a/mlir/include/mlir/Analysis/Presburger/Fraction.h b/mlir/include/mlir/Analysis/Presburger/Fraction.h
index f4f1be97147bf63..585a6d48506d245 100644
--- a/mlir/include/mlir/Analysis/Presburger/Fraction.h
+++ b/mlir/include/mlir/Analysis/Presburger/Fraction.h
@@ -15,6 +15,7 @@
 #define MLIR_ANALYSIS_PRESBURGER_FRACTION_H
 
 #include "llvm/ADT/DynamicAPInt.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 namespace presburger {
diff --git a/mlir/include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h b/mlir/include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h
new file mode 100644
index 000000000000000..e1dcedba094c03d
--- /dev/null
+++ b/mlir/include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h
@@ -0,0 +1,18 @@
+//===- AttrToSPIRVConverter.h - GPU attributes conversion to SPIR-V - C++ -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_CONVERSION_GPUCOMMON_ATTRTOSPIRVCONVERTER_H_
+#define MLIR_CONVERSION_GPUCOMMON_ATTRTOSPIRVCONVERTER_H_
+
+#include <mlir/Dialect/GPU/IR/GPUDialect.h>
+#include <mlir/Dialect/SPIRV/IR/SPIRVEnums.h>
+
+namespace mlir {
+spirv::StorageClass addressSpaceToStorageClass(gpu::AddressSpace addressSpace);
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_GPUCOMMON_ATTRTOSPIRVCONVERTER_H_
diff --git a/mlir/include/mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h b/mlir/include/mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h
index e156c3093e21be4..20c9f9c394525df 100644
--- a/mlir/include/mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h
+++ b/mlir/include/mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h
@@ -16,12 +16,17 @@ class DialectRegistry;
 class LLVMTypeConverter;
 class RewritePatternSet;
 class Pass;
+class TypeConverter;
 
 #define GEN_PASS_DECL_CONVERTGPUOPSTOLLVMSPVOPS
 #include "mlir/Conversion/Passes.h.inc"
 
 void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &converter,
                                             RewritePatternSet &patterns);
+
+/// Populates memory space attribute conversion rules for lowering
+/// gpu.address_space to integer values.
+void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter);
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_GPUTOLLVMSPV_GPUTOLLVMSPVPASS_H_
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index dddbd837c1cbc70..97e0580c8980800 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -402,6 +402,61 @@ def AMDGPU_RawBufferAtomicUminOp :
   let hasVerifier = 1;
 }
 
+def AMDGPU_DPPPerm : I32EnumAttr<"DPPPerm",
+    "The possible permutations for a DPP operation",
+    [  
+      I32EnumAttrCase<"quad_perm",  0>,
+      I32EnumAttrCase<"row_shl",    1>,
+      I32EnumAttrCase<"row_shr",    2>,
+      I32EnumAttrCase<"row_ror",    3>,
+      I32EnumAttrCase<"wave_shl",   4>,
+      I32EnumAttrCase<"wave_shr",   5>,
+      I32EnumAttrCase<"wave_ror",   6>,
+      I32EnumAttrCase<"wave_rol",   7>,
+      I32EnumAttrCase<"row_mirror", 8>,
+      I32EnumAttrCase<"row_half_mirror", 9>,
+      I32EnumAttrCase<"row_bcast_15", 10>,
+      I32EnumAttrCase<"row_bcast_31", 11>
+    ]> {
+  let genSpecializedAttr = 0;  
+  let cppNamespace = "::mlir::amdgpu";
+}
+
+def AMDGPU_DPPPermAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_DPPPerm,
+  "dpp_perm">;
+
+def AMDGPU_DPPOp : AMDGPU_Op<"dpp", [SameTypeOperands, AllTypesMatch<["result", "old", "src"]>]>,
+  Arguments<(ins AnyType:$old,
+                  AnyType:$src,
+                  AMDGPU_DPPPermAttr:$kind,
+                  OptionalAttr<AnyAttrOf<[I32Attr, ArrayAttr, UnitAttr]>>:$permArgument,
+                  DefaultValuedAttr<I32Attr, "0xf">:$row_mask,
+                  DefaultValuedAttr<I32Attr, "0xf">:$bank_mask,
+                  DefaultValuedAttr<BoolAttr, "false">:$bound_ctrl)> {
+  let summary = "AMDGPU DPP operation";
+  let description = [{
+    This operation represents DPP functionality in a GPU program.
+     DPP provides the following operations:
+    - Full crossbar in a group of four (`quad_perm`)
+    - Wavefront shift left by one lane (`wave_shl`)
+    - Wavefront shift right by one lane (`wave_shr`)
+    - Wavefront rotate right by one lane (`wave_ror`)
+    - Wavefront rotate left by one lane (`wave_rol`)
+    - Row shift left by 1–15 lanes (`row_shl`)
+    - Row shift right by 1–15 lanes (`row_shr`)
+    - Row rotate right by 1–15 lanes (`row_ror`)
+    - Reverse within a row (`row_mirror`)
+    - Reverse within a half-row (`row_half_mirror`)
+    - Broadcast the 15th lane of each row to the next row (`row_bcast`)
+    - Broadcast lane 31 to rows 2 and 3 (`row_bcast`)
+  }];
+  let results = (outs AnyType:$result);
+  let assemblyFormat = [{
+    $old $src $kind (`(` $permArgument^ `)`)? attr-dict `:` type($result)
+  }];
+  let hasVerifier = 1;
+}
+
 def AMDGPU_LDSBarrierOp : AMDGPU_Op<"lds_barrier"> {
   let summary = "Barrier that includes a wait for LDS memory operations.";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index 16fa58ce47013d5..40903b4e288ca62 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -1188,7 +1188,7 @@ def EmitC_AssignOp : EmitC_Op<"assign", []> {
 }
 
 def EmitC_YieldOp : EmitC_Op<"yield",
-      [Pure, Terminator, ParentOneOf<["ExpressionOp", "IfOp", "ForOp"]>]> {
+      [Pure, Terminator, ParentOneOf<["ExpressionOp", "IfOp", "ForOp", "SwitchOp"]>]> {
   let summary = "Block termination operation";
   let description = [{
     The `emitc.yield` terminates its parent EmitC op's region, optionally yielding
@@ -1302,5 +1302,87 @@ def EmitC_SubscriptOp : EmitC_Op<"subscript", []> {
   let assemblyFormat = "$value `[` $indices `]` attr-dict `:` functional-type(operands, results)";
 }
 
+def EmitC_SwitchOp : EmitC_Op<"switch", [RecursiveMemoryEffects,
+    SingleBlockImplicitTerminator<"emitc::YieldOp">,
+    DeclareOpInterfaceMethods<RegionBranchOpInterface,
+                              ["getRegionInvocationBounds",
+                               "getEntrySuccessorRegions"]>]> {
+  let summary = "Switch operation";
+  let description = [{
+    The `emitc.switch` is a control-flow operation that branches to one of
+    the given regions based on the values of the argument and the cases.
+    The operand to a switch operation is a opaque, integral or pointer
+    wide types.
+
+    The operation always has a "default" region and any number of case regions
+    denoted by integer constants. Control-flow transfers to the case region
+    whose constant value equals the value of the argument. If the argument does
+    not equal any of the case values, control-flow transfer to the "default"
+    region.
+
+    The operation does not return any value. Moreover, case regions must be
+    explicitly terminated using the `emitc.yield` operation. Default region is
+    yielded implicitly.
+
+    Example:
+
+    ```mlir
+    // Example:
+    emitc.switch %0 : i32
+    case 2 {
+      %1 = emitc.call_opaque "func_b" () : () -> i32
+      emitc.yield
+    }
+    case 5 {
+      %2 = emitc.call_opaque "func_a" () : () -> i32
+      emitc.yield
+    }
+    default {
+      %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+      emitc.call_opaque "func2" (%3) : (f32) -> ()
+    }
+    ```
+    ```c++
+    // Code emitted for the operations above.
+    switch (v1) {
+    case 2: {
+      int32_t v2 = func_b();
+      break;
+    }
+    case 5: {
+      int32_t v3 = func_a();
+      break;
+    }
+    default: {
+      float v4 = 4.200000000e+01f;
+      func2(v4);
+      break;
+    }
+    ```
+  }];
+
+  let arguments = (ins IntegerIndexOrOpaqueType:$arg, DenseI64ArrayAttr:$cases);
+  let results = (outs);
+  let regions = (region SizedRegion<1>:$defaultRegion,
+                        VariadicRegion<SizedRegion<1>>:$caseRegions);
+
+  let assemblyFormat = [{
+    $arg `:` type($arg) attr-dict custom<SwitchCases>($cases, $caseRegions) `\n`
+    `` `default` $defaultRegion
+  }];
+
+  let extraClassDeclaration = [{
+    /// Get the number of cases.
+    unsigned getNumCases();
+
+    /// Get the default region body.
+    Block &getDefaultBlock();
+
+    /// Get the body of a case region.
+    Block &getCaseBlock(unsigned idx);
+  }];
+
+  let hasVerifier = 1;
+}
 
 #endif // MLIR_DIALECT_EMITC_IR_EMITC
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 2e1635e590cad6a..86983e95fdd33d0 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -842,22 +842,27 @@ def LLVM_GetActiveLaneMaskOp
 /// Create a call to Masked Load intrinsic.
 def LLVM_MaskedLoadOp : LLVM_OneResultIntrOp<"masked.load"> {
   let arguments = (ins LLVM_AnyPointer:$data, LLVM_VectorOf<I1>:$mask,
-                   Variadic<LLVM_AnyVector>:$pass_thru, I32Attr:$alignment);
+                   Variadic<LLVM_AnyVector>:$pass_thru, I32Attr:$alignment,
+                   UnitAttr:$nontemporal);
   let results = (outs LLVM_AnyVector:$res);
   let assemblyFormat =
     "operands attr-dict `:` functional-type(operands, results)";
 
   string llvmBuilder = [{
-    $res = $pass_thru.empty() ? builder.CreateMaskedLoad(
+    auto *inst = $pass_thru.empty() ? builder.CreateMaskedLoad(
         $_resultType, $data, llvm::Align($alignment), $mask) :
       builder.CreateMaskedLoad(
         $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru[0]);
-  }];
+    $res = inst;
+  }] #setNonTemporalMetadataCode;
   string mlirBuilder = [{
+    auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
+    bool nontemporal = intrinInst->hasMetadata(llvm::LLVMContext::MD_nontemporal);
     $res = $_builder.create<LLVM::MaskedLoadOp>($_location,
-      $_resultType, $data, $mask, $pass_thru, $_int_attr($alignment));
+      $_resultType, $data, $mask, $pass_thru, $_int_attr($alignment),
+        nontemporal ? $_builder.getUnitAttr() : nullptr);
   }];
-  list<int> llvmArgIndices = [0, 2, 3, 1];
+  list<int> llvmArgIndices = [0, 2, 3, 1, -1];
 }
 
 /// Create a call to Masked Store intrinsic.
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index b81b7e577b213a0..a1e6fc3e299009c 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -555,6 +555,30 @@ def ROCDL_RawBufferAtomicUMinOp :
   let hasCustomAssemblyFormat = 1;
 }
 
+// DPP Update intrinsic
+def ROCDL_DPPUpdateOp : ROCDL_IntrOp<"update.dpp", [], [0],
+    [AllTypesMatch<["res", "src", "old"]>], 1>,
+  Arguments<(ins LLVM_Type:$old, LLVM_Type:$src, I32Attr:$dppCtrl, I32Attr:$rowMask,
+      I32Attr:$bankMask, I1Attr:$boundCtrl)> {
+  let results = (outs LLVM_Type:$res);
+  let assemblyFormat = [{
+    attr-dict $old `,` $src `with` $dppCtrl `,` $rowMask `,` $bankMask `,` $boundCtrl `:` type($src)
+  }];
+  string llvmBuilder = [{
+      auto vdataType = moduleTranslation.convertType(op.getSrc().getType());
+      llvm::Value *args[] = {
+        moduleTranslation.lookupValue(op.getOld()),
+        moduleTranslation.lookupValue(op.getSrc()),
+          builder.getInt32(op.getDppCtrl()),
+          builder.getInt32(op.getRowMask()),
+          builder.getInt32(op.getBankMask()),
+          builder.getInt1(op.getBoundCtrl())
+      };
+      $res = createIntrinsicCall(builder,
+        llvm::Intrinsic::amdgcn_update_dpp, args, {vdataType});
+  }];
+}
+
 //===---------------------------------------------------------------------===//
 // 8-bit float intrinsics
 //===---------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
index a9007c8db3078e9..5b6a90f806bedd9 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
@@ -154,8 +154,13 @@ def Linalg_SoftmaxOp : Linalg_Op<"softmax",
   let hasVerifier = 1;
 }
 
-def Linalg_WinogradFilterTransformOp :
-    Linalg_Op<"winograd_filter_transform", [AllElementTypesMatch<["filter", "output"]>]> {
+def Linalg_WinogradFilterTransformOp : Linalg_Op<"winograd_filter_transform",
+    [AllElementTypesMatch<["filter", "output"]>,
+     DeclareOpInterfaceMethods<TilingInterface,
+      ["getIterationDomain",
+       "getLoopIteratorTypes",
+       "getResultTilePosition",
+       "getTiledImplementation"]>]> {
   let summary = "Winograd filter transform operator";
   let description = [{
     Winograd Conv2D algorithm will convert linalg Conv2D operator into batched
@@ -190,11 +195,42 @@ def Linalg_WinogradFilterTransformOp :
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
   }];
+  let extraClassDeclaration = [{
+    ShapedType getFilterOperandType() {
+      return cast<ShapedType>(getFilter().getType());
+    }
+    ShapedType getOutputOperandType() {
+      return cast<ShapedType>(getOutput().getType());
+    }
+    int64_t getFilterOperandRank() {
+      return getFilterOperandType().getRank();
+    }
+    int64_t getOutputOperandRank() {
+      return getOutputOperandType().getRank();
+    }
+    int64_t getFilterFDim() {
+      return 0;
+    }
+    int64_t getFilterHDim() {
+      return 1;
+    }
+    int64_t getFilterWDim() {
+      return 2;
+    }
+    int64_t getFilterCDim() {
+      return 3;
+    }
+  }];
   let hasVerifier = 1;
 }
 
-def Linalg_WinogradInputTransformOp :
-    Linalg_Op<"winograd_input_transform", [AllElementTypesMatch<["input", "output"]>]> {
+def Linalg_WinogradInputTransformOp : Linalg_Op<"winograd_input_transform",
+    [AllElementTypesMatch<["input", "output"]>,
+     DeclareOpInterfaceMethods<TilingInterface,
+      ["getIterationDomain",
+       "getLoopIteratorTypes",
+       "getResultTilePosition",
+       "getTiledImplementation"]>]> {
   let summary = "Winograd input transform operator";
   let description = [{
     Winograd Conv2D algorithm will convert linalg Conv2D operator into batched
@@ -229,11 +265,60 @@ def Linalg_WinogradInputTransformOp :
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
   }];
+  let extraClassDeclaration = [{
+    ShapedType getInputOperandType() {
+      return cast<ShapedType>(getInput().getType());
+    }
+    ShapedType getOutputOperandType() {
+      return cast<ShapedType>(getOutput().getType());
+    }
+    int64_t getInputOperandRank() {
+      return getInputOperandType().getRank();
+    }
+    int64_t getOutputOperandRank() {
+      return getOutputOperandType().getRank();
+    }
+    int64_t getInputNDim() {
+      return 0;
+    }
+    int64_t getInputHDim() {
+      return 1;
+    }
+    int64_t getInputWDim() {
+      return 2;
+    }
+    int64_t getInputCDim() {
+      return 3;
+    }
+    int64_t getOutputAlphaHDim() {
+      return 0;
+    }
+    int64_t getOutputAlphaWDim() {
+      return 1;
+    }
+    int64_t getOutputTileHDim() {
+      return 2;
+    }
+    int64_t getOutputTileWDim() {
+      return 3;
+    }
+    int64_t getOutputNDim() {
+      return 4;
+    }
+    int64_t getOutputCDim() {
+      return 5;
+    }
+  }];
   let hasVerifier = 1;
 }
 
-def Linalg_WinogradOutputTransformOp :
-    Linalg_Op<"winograd_output_transform", [AllElementTypesMatch<["value", "output"]>]> {
+def Linalg_WinogradOutputTransformOp : Linalg_Op<"winograd_output_transform",
+    [AllElementTypesMatch<["value", "output"]>,
+     DeclareOpInterfaceMethods<TilingInterface,
+      ["getIterationDomain",
+       "getLoopIteratorTypes",
+       "getResultTilePosition",
+       "getTiledImplementation"]>]> {
   let summary = "Winograd output transform operator";
   let description = [{
     Winograd Conv2D algorithm will convert linalg Conv2D operator into batched
@@ -268,6 +353,50 @@ def Linalg_WinogradOutputTransformOp :
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
   }];
+  let extraClassDeclaration = [{
+    ShapedType getValueOperandType() {
+      return cast<ShapedType>(getValue().getType());
+    }
+    ShapedType getOutputOperandType() {
+      return cast<ShapedType>(getOutput().getType());
+    }
+    int64_t getValueOperandRank() {
+      return getValueOperandType().getRank();
+    }
+    int64_t getOutputOperandRank() {
+      return getOutputOperandType().getRank();
+    }
+    int64_t getValueAlphaHDim() {
+      return 0;
+    }
+    int64_t getValueAlphaWDim() {
+      return 1;
+    }
+    int64_t getValueTileHDim() {
+      return 2;
+    }
+    int64_t getValueTileWDim() {
+      return 3;
+    }
+    int64_t getValueNDim() {
+      return 4;
+    }
+    int64_t getValueFDim() {
+      return 5;
+    }
+    int64_t getOutputNDim() {
+      return 0;
+    }
+    int64_t getOutputHDim() {
+      return 1;
+    }
+    int64_t getOutputWDim() {
+      return 2;
+    }
+    int64_t getOutputFDim() {
+      return 3;
+    }
+  }];
   let hasVerifier = 1;
 }
 
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index ecc86999006db64..106f0d79d9792d6 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -2697,4 +2697,41 @@ def WinogradConv2DOp : Op<Transform_Dialect,
   }];
 }
 
+def DecomposeWinogradOp : Op<Transform_Dialect,
+    "structured.decompose_winograd_op",
+    [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
+     TransformOpInterface, TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
+  let description = [{
+    Decompose winograd operations. It will convert filter, input and output
+    transform operations into a combination of scf, tensor, and linalg
+    equivalent operations. Before applying this transform operations, users
+    need to tile winograd transform operations into supported sizes.
+
+    #### Return modes:
+
+    This operation fails if `target` is unsupported. Otherwise, the operation
+    succeeds and returns a handle of the sequence that replaces the original
+    operations.
+  }];
+
+  let arguments = (ins TransformHandleTypeInterface:$target);
+  let results = (outs TransformHandleTypeInterface:$transformed);
+
+  let assemblyFormat =
+    "$target attr-dict `:` functional-type($target, results)";
+
+  let builders = [
+    OpBuilder<(ins "Value":$target)>
+  ];
+
+  let extraClassDeclaration = [{
+    ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
+        ::mlir::Operation *target,
+        ::mlir::transform::ApplyToEachResultList &results,
+        ::mlir::transform::TransformState &state);
+  }];
+}
+
 #endif // LINALG_TRANSFORM_OPS
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 477ef7bfafb181f..861e14d22d96251 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -1316,6 +1316,63 @@ FailureOr<Operation *> winogradConv2D(RewriterBase &rewriter,
                                       linalg::Conv2DNhwcFhwcOp op, int64_t m,
                                       int64_t r);
 
+/// Rewrite linalg.winograd_filter_transform. The data layout of the filter is
+/// FHWC. The transformation matrix is 2-dimension. We need to extract H x W
+/// from FHWC first. We generate 2 levels of loops to iterate on F and C. After
+/// the rewriting, we get
+///
+/// scf.for %f = lo_f to hi_f step 1
+///   scf.for %c = lo_c to hi_c step 1
+///     %extracted = extract filter<h x w> from filter<f x h x w x c>
+///     %ret = linalg.matmul G, %extracted
+///     %ret = linalg.matmul %ret, GT
+///     %inserted = insert %ret into filter<h x w x c x f>
+FailureOr<Operation *>
+decomposeWinogradFilterTransformOp(RewriterBase &rewriter,
+                                   linalg::WinogradFilterTransformOp op);
+
+/// Rewrite linalg.winograd_input_transform. The data layout of the input is
+/// NHWC. The transformation matrix is 2-dimension. We need to extract H x W
+/// from NHWC first. We generate 4 levels of loops to iterate on N, C, tileH,
+/// and tileW. After the rewriting, we get
+///
+/// scf.for %h = 0 to tileH step 1
+///   scf.for %w = 0 to tileW step 1
+///     scf.for %n = 0 to N step 1
+///       scf.for %c = 0 to C step 1
+///         %extracted = extract %extracted<alphaH x alphaW> from
+///                              %input<N x H x W x C>
+///                              at [%n, (%h x m), (%w x m), %c]
+///         %ret = linalg.matmul BT, %extracted
+///         %ret = linalg.matmul %ret, B
+///         %inserted = insert %ret<alphaH x alphaW> into
+///                            %output<alphaH x alphaW x tileH x tileW x N x C>
+///                            at [0, 0, %h, %w, %n, %c]
+FailureOr<Operation *>
+decomposeWinogradInputTransformOp(RewriterBase &rewriter,
+                                  linalg::WinogradInputTransformOp op);
+
+/// Rewrite linalg.winograd_output_transform. The data layout of the output is
+/// HWNF. The transformation matrix is 2-dimension. We need to extract H x W
+/// from HWNF first. We generate 4 levels of loops to iterate on N, F, tileH,
+/// and tileW. After the transformation, we get
+///
+/// scf.for %h = 0 to tileH step 1
+///   scf.for %w = 0 to tileW step 1
+///     scf.for %n = 0 to N step 1
+///       scf.for %f = 0 to F step 1
+///         %extracted = extract %extracted<alphaH x alphaW> from
+///                              %input<alphaH x alphaW x tileH x tileW x N x F>
+///                              at [0, 0, %h, %w, %n, %f]
+///         %ret = linalg.matmul AT, %extracted
+///         %ret = linalg.matmul %ret, A
+///         %inserted = insert %ret<alphaH x alphaW> into
+///                            output<N x H x W x F>
+///                            at [%n, (%h x m), (%w x m), %f]
+FailureOr<Operation *>
+decomposeWinogradOutputTransformOp(RewriterBase &rewriter,
+                                   linalg::WinogradOutputTransformOp op);
+
 //===----------------------------------------------------------------------===//
 // Rewrite patterns wrapping transformations.
 // TODO: every single such pattern should be a close to noop wrapper around a
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
index af0b2624feb327f..3b7da9b44a08fbf 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
@@ -4243,230 +4243,231 @@ class SPIRV_OpCode<string name, int val> {
 
 // Begin opcode section. Generated from SPIR-V spec; DO NOT MODIFY!
 
-def SPIRV_OC_OpNop                        : I32EnumAttrCase<"OpNop", 0>;
-def SPIRV_OC_OpUndef                      : I32EnumAttrCase<"OpUndef", 1>;
-def SPIRV_OC_OpSourceContinued            : I32EnumAttrCase<"OpSourceContinued", 2>;
-def SPIRV_OC_OpSource                     : I32EnumAttrCase<"OpSource", 3>;
-def SPIRV_OC_OpSourceExtension            : I32EnumAttrCase<"OpSourceExtension", 4>;
-def SPIRV_OC_OpName                       : I32EnumAttrCase<"OpName", 5>;
-def SPIRV_OC_OpMemberName                 : I32EnumAttrCase<"OpMemberName", 6>;
-def SPIRV_OC_OpString                     : I32EnumAttrCase<"OpString", 7>;
-def SPIRV_OC_OpLine                       : I32EnumAttrCase<"OpLine", 8>;
-def SPIRV_OC_OpExtension                  : I32EnumAttrCase<"OpExtension", 10>;
-def SPIRV_OC_OpExtInstImport              : I32EnumAttrCase<"OpExtInstImport", 11>;
-def SPIRV_OC_OpExtInst                    : I32EnumAttrCase<"OpExtInst", 12>;
-def SPIRV_OC_OpMemoryModel                : I32EnumAttrCase<"OpMemoryModel", 14>;
-def SPIRV_OC_OpEntryPoint                 : I32EnumAttrCase<"OpEntryPoint", 15>;
-def SPIRV_OC_OpExecutionMode              : I32EnumAttrCase<"OpExecutionMode", 16>;
-def SPIRV_OC_OpCapability                 : I32EnumAttrCase<"OpCapability", 17>;
-def SPIRV_OC_OpTypeVoid                   : I32EnumAttrCase<"OpTypeVoid", 19>;
-def SPIRV_OC_OpTypeBool                   : I32EnumAttrCase<"OpTypeBool", 20>;
-def SPIRV_OC_OpTypeInt                    : I32EnumAttrCase<"OpTypeInt", 21>;
-def SPIRV_OC_OpTypeFloat                  : I32EnumAttrCase<"OpTypeFloat", 22>;
-def SPIRV_OC_OpTypeVector                 : I32EnumAttrCase<"OpTypeVector", 23>;
-def SPIRV_OC_OpTypeMatrix                 : I32EnumAttrCase<"OpTypeMatrix", 24>;
-def SPIRV_OC_OpTypeImage                  : I32EnumAttrCase<"OpTypeImage", 25>;
-def SPIRV_OC_OpTypeSampledImage           : I32EnumAttrCase<"OpTypeSampledImage", 27>;
-def SPIRV_OC_OpTypeArray                  : I32EnumAttrCase<"OpTypeArray", 28>;
-def SPIRV_OC_OpTypeRuntimeArray           : I32EnumAttrCase<"OpTypeRuntimeArray", 29>;
-def SPIRV_OC_OpTypeStruct                 : I32EnumAttrCase<"OpTypeStruct", 30>;
-def SPIRV_OC_OpTypePointer                : I32EnumAttrCase<"OpTypePointer", 32>;
-def SPIRV_OC_OpTypeFunction               : I32EnumAttrCase<"OpTypeFunction", 33>;
-def SPIRV_OC_OpTypeForwardPointer         : I32EnumAttrCase<"OpTypeForwardPointer", 39>;
-def SPIRV_OC_OpConstantTrue               : I32EnumAttrCase<"OpConstantTrue", 41>;
-def SPIRV_OC_OpConstantFalse              : I32EnumAttrCase<"OpConstantFalse", 42>;
-def SPIRV_OC_OpConstant                   : I32EnumAttrCase<"OpConstant", 43>;
-def SPIRV_OC_OpConstantComposite          : I32EnumAttrCase<"OpConstantComposite", 44>;
-def SPIRV_OC_OpConstantNull               : I32EnumAttrCase<"OpConstantNull", 46>;
-def SPIRV_OC_OpSpecConstantTrue           : I32EnumAttrCase<"OpSpecConstantTrue", 48>;
-def SPIRV_OC_OpSpecConstantFalse          : I32EnumAttrCase<"OpSpecConstantFalse", 49>;
-def SPIRV_OC_OpSpecConstant               : I32EnumAttrCase<"OpSpecConstant", 50>;
-def SPIRV_OC_OpSpecConstantComposite      : I32EnumAttrCase<"OpSpecConstantComposite", 51>;
-def SPIRV_OC_OpSpecConstantOp             : I32EnumAttrCase<"OpSpecConstantOp", 52>;
-def SPIRV_OC_OpFunction                   : I32EnumAttrCase<"OpFunction", 54>;
-def SPIRV_OC_OpFunctionParameter          : I32EnumAttrCase<"OpFunctionParameter", 55>;
-def SPIRV_OC_OpFunctionEnd                : I32EnumAttrCase<"OpFunctionEnd", 56>;
-def SPIRV_OC_OpFunctionCall               : I32EnumAttrCase<"OpFunctionCall", 57>;
-def SPIRV_OC_OpVariable                   : I32EnumAttrCase<"OpVariable", 59>;
-def SPIRV_OC_OpLoad                       : I32EnumAttrCase<"OpLoad", 61>;
-def SPIRV_OC_OpStore                      : I32EnumAttrCase<"OpStore", 62>;
-def SPIRV_OC_OpCopyMemory                 : I32EnumAttrCase<"OpCopyMemory", 63>;
-def SPIRV_OC_OpAccessChain                : I32EnumAttrCase<"OpAccessChain", 65>;
-def SPIRV_OC_OpPtrAccessChain             : I32EnumAttrCase<"OpPtrAccessChain", 67>;
-def SPIRV_OC_OpInBoundsPtrAccessChain     : I32EnumAttrCase<"OpInBoundsPtrAccessChain", 70>;
-def SPIRV_OC_OpDecorate                   : I32EnumAttrCase<"OpDecorate", 71>;
-def SPIRV_OC_OpMemberDecorate             : I32EnumAttrCase<"OpMemberDecorate", 72>;
-def SPIRV_OC_OpVectorExtractDynamic       : I32EnumAttrCase<"OpVectorExtractDynamic", 77>;
-def SPIRV_OC_OpVectorInsertDynamic        : I32EnumAttrCase<"OpVectorInsertDynamic", 78>;
-def SPIRV_OC_OpVectorShuffle              : I32EnumAttrCase<"OpVectorShuffle", 79>;
-def SPIRV_OC_OpCompositeConstruct         : I32EnumAttrCase<"OpCompositeConstruct", 80>;
-def SPIRV_OC_OpCompositeExtract           : I32EnumAttrCase<"OpCompositeExtract", 81>;
-def SPIRV_OC_OpCompositeInsert            : I32EnumAttrCase<"OpCompositeInsert", 82>;
-def SPIRV_OC_OpTranspose                  : I32EnumAttrCase<"OpTranspose", 84>;
-def SPIRV_OC_OpImageDrefGather            : I32EnumAttrCase<"OpImageDrefGather", 97>;
-def SPIRV_OC_OpImage                      : I32EnumAttrCase<"OpImage", 100>;
-def SPIRV_OC_OpImageQuerySize             : I32EnumAttrCase<"OpImageQuerySize", 104>;
-def SPIRV_OC_OpConvertFToU                : I32EnumAttrCase<"OpConvertFToU", 109>;
-def SPIRV_OC_OpConvertFToS                : I32EnumAttrCase<"OpConvertFToS", 110>;
-def SPIRV_OC_OpConvertSToF                : I32EnumAttrCase<"OpConvertSToF", 111>;
-def SPIRV_OC_OpConvertUToF                : I32EnumAttrCase<"OpConvertUToF", 112>;
-def SPIRV_OC_OpUConvert                   : I32EnumAttrCase<"OpUConvert", 113>;
-def SPIRV_OC_OpSConvert                   : I32EnumAttrCase<"OpSConvert", 114>;
-def SPIRV_OC_OpFConvert                   : I32EnumAttrCase<"OpFConvert", 115>;
-def SPIRV_OC_OpConvertPtrToU              : I32EnumAttrCase<"OpConvertPtrToU", 117>;
-def SPIRV_OC_OpConvertUToPtr              : I32EnumAttrCase<"OpConvertUToPtr", 120>;
-def SPIRV_OC_OpPtrCastToGeneric           : I32EnumAttrCase<"OpPtrCastToGeneric", 121>;
-def SPIRV_OC_OpGenericCastToPtr           : I32EnumAttrCase<"OpGenericCastToPtr", 122>;
-def SPIRV_OC_OpGenericCastToPtrExplicit   : I32EnumAttrCase<"OpGenericCastToPtrExplicit", 123>;
-def SPIRV_OC_OpBitcast                    : I32EnumAttrCase<"OpBitcast", 124>;
-def SPIRV_OC_OpSNegate                    : I32EnumAttrCase<"OpSNegate", 126>;
-def SPIRV_OC_OpFNegate                    : I32EnumAttrCase<"OpFNegate", 127>;
-def SPIRV_OC_OpIAdd                       : I32EnumAttrCase<"OpIAdd", 128>;
-def SPIRV_OC_OpFAdd                       : I32EnumAttrCase<"OpFAdd", 129>;
-def SPIRV_OC_OpISub                       : I32EnumAttrCase<"OpISub", 130>;
-def SPIRV_OC_OpFSub                       : I32EnumAttrCase<"OpFSub", 131>;
-def SPIRV_OC_OpIMul                       : I32EnumAttrCase<"OpIMul", 132>;
-def SPIRV_OC_OpFMul                       : I32EnumAttrCase<"OpFMul", 133>;
-def SPIRV_OC_OpUDiv                       : I32EnumAttrCase<"OpUDiv", 134>;
-def SPIRV_OC_OpSDiv                       : I32EnumAttrCase<"OpSDiv", 135>;
-def SPIRV_OC_OpFDiv                       : I32EnumAttrCase<"OpFDiv", 136>;
-def SPIRV_OC_OpUMod                       : I32EnumAttrCase<"OpUMod", 137>;
-def SPIRV_OC_OpSRem                       : I32EnumAttrCase<"OpSRem", 138>;
-def SPIRV_OC_OpSMod                       : I32EnumAttrCase<"OpSMod", 139>;
-def SPIRV_OC_OpFRem                       : I32EnumAttrCase<"OpFRem", 140>;
-def SPIRV_OC_OpFMod                       : I32EnumAttrCase<"OpFMod", 141>;
-def SPIRV_OC_OpVectorTimesScalar          : I32EnumAttrCase<"OpVectorTimesScalar", 142>;
-def SPIRV_OC_OpMatrixTimesScalar          : I32EnumAttrCase<"OpMatrixTimesScalar", 143>;
-def SPIRV_OC_OpMatrixTimesMatrix          : I32EnumAttrCase<"OpMatrixTimesMatrix", 146>;
-def SPIRV_OC_OpDot                        : I32EnumAttrCase<"OpDot", 148>;
-def SPIRV_OC_OpIAddCarry                  : I32EnumAttrCase<"OpIAddCarry", 149>;
-def SPIRV_OC_OpISubBorrow                 : I32EnumAttrCase<"OpISubBorrow", 150>;
-def SPIRV_OC_OpUMulExtended               : I32EnumAttrCase<"OpUMulExtended", 151>;
-def SPIRV_OC_OpSMulExtended               : I32EnumAttrCase<"OpSMulExtended", 152>;
-def SPIRV_OC_OpIsNan                      : I32EnumAttrCase<"OpIsNan", 156>;
-def SPIRV_OC_OpIsInf                      : I32EnumAttrCase<"OpIsInf", 157>;
-def SPIRV_OC_OpOrdered                    : I32EnumAttrCase<"OpOrdered", 162>;
-def SPIRV_OC_OpUnordered                  : I32EnumAttrCase<"OpUnordered", 163>;
-def SPIRV_OC_OpLogicalEqual               : I32EnumAttrCase<"OpLogicalEqual", 164>;
-def SPIRV_OC_OpLogicalNotEqual            : I32EnumAttrCase<"OpLogicalNotEqual", 165>;
-def SPIRV_OC_OpLogicalOr                  : I32EnumAttrCase<"OpLogicalOr", 166>;
-def SPIRV_OC_OpLogicalAnd                 : I32EnumAttrCase<"OpLogicalAnd", 167>;
-def SPIRV_OC_OpLogicalNot                 : I32EnumAttrCase<"OpLogicalNot", 168>;
-def SPIRV_OC_OpSelect                     : I32EnumAttrCase<"OpSelect", 169>;
-def SPIRV_OC_OpIEqual                     : I32EnumAttrCase<"OpIEqual", 170>;
-def SPIRV_OC_OpINotEqual                  : I32EnumAttrCase<"OpINotEqual", 171>;
-def SPIRV_OC_OpUGreaterThan               : I32EnumAttrCase<"OpUGreaterThan", 172>;
-def SPIRV_OC_OpSGreaterThan               : I32EnumAttrCase<"OpSGreaterThan", 173>;
-def SPIRV_OC_OpUGreaterThanEqual          : I32EnumAttrCase<"OpUGreaterThanEqual", 174>;
-def SPIRV_OC_OpSGreaterThanEqual          : I32EnumAttrCase<"OpSGreaterThanEqual", 175>;
-def SPIRV_OC_OpULessThan                  : I32EnumAttrCase<"OpULessThan", 176>;
-def SPIRV_OC_OpSLessThan                  : I32EnumAttrCase<"OpSLessThan", 177>;
-def SPIRV_OC_OpULessThanEqual             : I32EnumAttrCase<"OpULessThanEqual", 178>;
-def SPIRV_OC_OpSLessThanEqual             : I32EnumAttrCase<"OpSLessThanEqual", 179>;
-def SPIRV_OC_OpFOrdEqual                  : I32EnumAttrCase<"OpFOrdEqual", 180>;
-def SPIRV_OC_OpFUnordEqual                : I32EnumAttrCase<"OpFUnordEqual", 181>;
-def SPIRV_OC_OpFOrdNotEqual               : I32EnumAttrCase<"OpFOrdNotEqual", 182>;
-def SPIRV_OC_OpFUnordNotEqual             : I32EnumAttrCase<"OpFUnordNotEqual", 183>;
-def SPIRV_OC_OpFOrdLessThan               : I32EnumAttrCase<"OpFOrdLessThan", 184>;
-def SPIRV_OC_OpFUnordLessThan             : I32EnumAttrCase<"OpFUnordLessThan", 185>;
-def SPIRV_OC_OpFOrdGreaterThan            : I32EnumAttrCase<"OpFOrdGreaterThan", 186>;
-def SPIRV_OC_OpFUnordGreaterThan          : I32EnumAttrCase<"OpFUnordGreaterThan", 187>;
-def SPIRV_OC_OpFOrdLessThanEqual          : I32EnumAttrCase<"OpFOrdLessThanEqual", 188>;
-def SPIRV_OC_OpFUnordLessThanEqual        : I32EnumAttrCase<"OpFUnordLessThanEqual", 189>;
-def SPIRV_OC_OpFOrdGreaterThanEqual       : I32EnumAttrCase<"OpFOrdGreaterThanEqual", 190>;
-def SPIRV_OC_OpFUnordGreaterThanEqual     : I32EnumAttrCase<"OpFUnordGreaterThanEqual", 191>;
-def SPIRV_OC_OpShiftRightLogical          : I32EnumAttrCase<"OpShiftRightLogical", 194>;
-def SPIRV_OC_OpShiftRightArithmetic       : I32EnumAttrCase<"OpShiftRightArithmetic", 195>;
-def SPIRV_OC_OpShiftLeftLogical           : I32EnumAttrCase<"OpShiftLeftLogical", 196>;
-def SPIRV_OC_OpBitwiseOr                  : I32EnumAttrCase<"OpBitwiseOr", 197>;
-def SPIRV_OC_OpBitwiseXor                 : I32EnumAttrCase<"OpBitwiseXor", 198>;
-def SPIRV_OC_OpBitwiseAnd                 : I32EnumAttrCase<"OpBitwiseAnd", 199>;
-def SPIRV_OC_OpNot                        : I32EnumAttrCase<"OpNot", 200>;
-def SPIRV_OC_OpBitFieldInsert             : I32EnumAttrCase<"OpBitFieldInsert", 201>;
-def SPIRV_OC_OpBitFieldSExtract           : I32EnumAttrCase<"OpBitFieldSExtract", 202>;
-def SPIRV_OC_OpBitFieldUExtract           : I32EnumAttrCase<"OpBitFieldUExtract", 203>;
-def SPIRV_OC_OpBitReverse                 : I32EnumAttrCase<"OpBitReverse", 204>;
-def SPIRV_OC_OpBitCount                   : I32EnumAttrCase<"OpBitCount", 205>;
-def SPIRV_OC_OpControlBarrier             : I32EnumAttrCase<"OpControlBarrier", 224>;
-def SPIRV_OC_OpMemoryBarrier              : I32EnumAttrCase<"OpMemoryBarrier", 225>;
-def SPIRV_OC_OpAtomicExchange             : I32EnumAttrCase<"OpAtomicExchange", 229>;
-def SPIRV_OC_OpAtomicCompareExchange      : I32EnumAttrCase<"OpAtomicCompareExchange", 230>;
-def SPIRV_OC_OpAtomicCompareExchangeWeak  : I32EnumAttrCase<"OpAtomicCompareExchangeWeak", 231>;
-def SPIRV_OC_OpAtomicIIncrement           : I32EnumAttrCase<"OpAtomicIIncrement", 232>;
-def SPIRV_OC_OpAtomicIDecrement           : I32EnumAttrCase<"OpAtomicIDecrement", 233>;
-def SPIRV_OC_OpAtomicIAdd                 : I32EnumAttrCase<"OpAtomicIAdd", 234>;
-def SPIRV_OC_OpAtomicISub                 : I32EnumAttrCase<"OpAtomicISub", 235>;
-def SPIRV_OC_OpAtomicSMin                 : I32EnumAttrCase<"OpAtomicSMin", 236>;
-def SPIRV_OC_OpAtomicUMin                 : I32EnumAttrCase<"OpAtomicUMin", 237>;
-def SPIRV_OC_OpAtomicSMax                 : I32EnumAttrCase<"OpAtomicSMax", 238>;
-def SPIRV_OC_OpAtomicUMax                 : I32EnumAttrCase<"OpAtomicUMax", 239>;
-def SPIRV_OC_OpAtomicAnd                  : I32EnumAttrCase<"OpAtomicAnd", 240>;
-def SPIRV_OC_OpAtomicOr                   : I32EnumAttrCase<"OpAtomicOr", 241>;
-def SPIRV_OC_OpAtomicXor                  : I32EnumAttrCase<"OpAtomicXor", 242>;
-def SPIRV_OC_OpPhi                        : I32EnumAttrCase<"OpPhi", 245>;
-def SPIRV_OC_OpLoopMerge                  : I32EnumAttrCase<"OpLoopMerge", 246>;
-def SPIRV_OC_OpSelectionMerge             : I32EnumAttrCase<"OpSelectionMerge", 247>;
-def SPIRV_OC_OpLabel                      : I32EnumAttrCase<"OpLabel", 248>;
-def SPIRV_OC_OpBranch                     : I32EnumAttrCase<"OpBranch", 249>;
-def SPIRV_OC_OpBranchConditional          : I32EnumAttrCase<"OpBranchConditional", 250>;
-def SPIRV_OC_OpReturn                     : I32EnumAttrCase<"OpReturn", 253>;
-def SPIRV_OC_OpReturnValue                : I32EnumAttrCase<"OpReturnValue", 254>;
-def SPIRV_OC_OpUnreachable                : I32EnumAttrCase<"OpUnreachable", 255>;
-def SPIRV_OC_OpGroupBroadcast             : I32EnumAttrCase<"OpGroupBroadcast", 263>;
-def SPIRV_OC_OpGroupIAdd                  : I32EnumAttrCase<"OpGroupIAdd", 264>;
-def SPIRV_OC_OpGroupFAdd                  : I32EnumAttrCase<"OpGroupFAdd", 265>;
-def SPIRV_OC_OpGroupFMin                  : I32EnumAttrCase<"OpGroupFMin", 266>;
-def SPIRV_OC_OpGroupUMin                  : I32EnumAttrCase<"OpGroupUMin", 267>;
-def SPIRV_OC_OpGroupSMin                  : I32EnumAttrCase<"OpGroupSMin", 268>;
-def SPIRV_OC_OpGroupFMax                  : I32EnumAttrCase<"OpGroupFMax", 269>;
-def SPIRV_OC_OpGroupUMax                  : I32EnumAttrCase<"OpGroupUMax", 270>;
-def SPIRV_OC_OpGroupSMax                  : I32EnumAttrCase<"OpGroupSMax", 271>;
-def SPIRV_OC_OpNoLine                     : I32EnumAttrCase<"OpNoLine", 317>;
-def SPIRV_OC_OpModuleProcessed            : I32EnumAttrCase<"OpModuleProcessed", 330>;
-def SPIRV_OC_OpGroupNonUniformElect       : I32EnumAttrCase<"OpGroupNonUniformElect", 333>;
-def SPIRV_OC_OpGroupNonUniformBroadcast   : I32EnumAttrCase<"OpGroupNonUniformBroadcast", 337>;
-def SPIRV_OC_OpGroupNonUniformBallot      : I32EnumAttrCase<"OpGroupNonUniformBallot", 339>;
-def SPIRV_OC_OpGroupNonUniformShuffle     : I32EnumAttrCase<"OpGroupNonUniformShuffle", 345>;
-def SPIRV_OC_OpGroupNonUniformShuffleXor  : I32EnumAttrCase<"OpGroupNonUniformShuffleXor", 346>;
-def SPIRV_OC_OpGroupNonUniformShuffleUp   : I32EnumAttrCase<"OpGroupNonUniformShuffleUp", 347>;
-def SPIRV_OC_OpGroupNonUniformShuffleDown : I32EnumAttrCase<"OpGroupNonUniformShuffleDown", 348>;
-def SPIRV_OC_OpGroupNonUniformIAdd        : I32EnumAttrCase<"OpGroupNonUniformIAdd", 349>;
-def SPIRV_OC_OpGroupNonUniformFAdd        : I32EnumAttrCase<"OpGroupNonUniformFAdd", 350>;
-def SPIRV_OC_OpGroupNonUniformIMul        : I32EnumAttrCase<"OpGroupNonUniformIMul", 351>;
-def SPIRV_OC_OpGroupNonUniformFMul        : I32EnumAttrCase<"OpGroupNonUniformFMul", 352>;
-def SPIRV_OC_OpGroupNonUniformSMin        : I32EnumAttrCase<"OpGroupNonUniformSMin", 353>;
-def SPIRV_OC_OpGroupNonUniformUMin        : I32EnumAttrCase<"OpGroupNonUniformUMin", 354>;
-def SPIRV_OC_OpGroupNonUniformFMin        : I32EnumAttrCase<"OpGroupNonUniformFMin", 355>;
-def SPIRV_OC_OpGroupNonUniformSMax        : I32EnumAttrCase<"OpGroupNonUniformSMax", 356>;
-def SPIRV_OC_OpGroupNonUniformUMax        : I32EnumAttrCase<"OpGroupNonUniformUMax", 357>;
-def SPIRV_OC_OpGroupNonUniformFMax        : I32EnumAttrCase<"OpGroupNonUniformFMax", 358>;
-def SPIRV_OC_OpGroupNonUniformBitwiseAnd  : I32EnumAttrCase<"OpGroupNonUniformBitwiseAnd", 359>;
-def SPIRV_OC_OpGroupNonUniformBitwiseOr   : I32EnumAttrCase<"OpGroupNonUniformBitwiseOr", 360>;
-def SPIRV_OC_OpGroupNonUniformBitwiseXor  : I32EnumAttrCase<"OpGroupNonUniformBitwiseXor", 361>;
-def SPIRV_OC_OpGroupNonUniformLogicalAnd  : I32EnumAttrCase<"OpGroupNonUniformLogicalAnd", 362>;
-def SPIRV_OC_OpGroupNonUniformLogicalOr   : I32EnumAttrCase<"OpGroupNonUniformLogicalOr", 363>;
-def SPIRV_OC_OpGroupNonUniformLogicalXor  : I32EnumAttrCase<"OpGroupNonUniformLogicalXor", 364>;
-def SPIRV_OC_OpSubgroupBallotKHR          : I32EnumAttrCase<"OpSubgroupBallotKHR", 4421>;
-def SPIRV_OC_OpSDot                       : I32EnumAttrCase<"OpSDot", 4450>;
-def SPIRV_OC_OpUDot                       : I32EnumAttrCase<"OpUDot", 4451>;
-def SPIRV_OC_OpSUDot                      : I32EnumAttrCase<"OpSUDot", 4452>;
-def SPIRV_OC_OpSDotAccSat                 : I32EnumAttrCase<"OpSDotAccSat", 4453>;
-def SPIRV_OC_OpUDotAccSat                 : I32EnumAttrCase<"OpUDotAccSat", 4454>;
-def SPIRV_OC_OpSUDotAccSat                : I32EnumAttrCase<"OpSUDotAccSat", 4455>;
-def SPIRV_OC_OpTypeCooperativeMatrixKHR   : I32EnumAttrCase<"OpTypeCooperativeMatrixKHR", 4456>;
-def SPIRV_OC_OpCooperativeMatrixLoadKHR   : I32EnumAttrCase<"OpCooperativeMatrixLoadKHR", 4457>;
-def SPIRV_OC_OpCooperativeMatrixStoreKHR  : I32EnumAttrCase<"OpCooperativeMatrixStoreKHR", 4458>;
-def SPIRV_OC_OpCooperativeMatrixMulAddKHR : I32EnumAttrCase<"OpCooperativeMatrixMulAddKHR", 4459>;
-def SPIRV_OC_OpCooperativeMatrixLengthKHR : I32EnumAttrCase<"OpCooperativeMatrixLengthKHR", 4460>;
-def SPIRV_OC_OpSubgroupBlockReadINTEL     : I32EnumAttrCase<"OpSubgroupBlockReadINTEL", 5575>;
-def SPIRV_OC_OpSubgroupBlockWriteINTEL    : I32EnumAttrCase<"OpSubgroupBlockWriteINTEL", 5576>;
-def SPIRV_OC_OpAssumeTrueKHR              : I32EnumAttrCase<"OpAssumeTrueKHR", 5630>;
-def SPIRV_OC_OpAtomicFAddEXT              : I32EnumAttrCase<"OpAtomicFAddEXT", 6035>;
-def SPIRV_OC_OpGroupIMulKHR               : I32EnumAttrCase<"OpGroupIMulKHR", 6401>;
-def SPIRV_OC_OpGroupFMulKHR               : I32EnumAttrCase<"OpGroupFMulKHR", 6402>;
-
-def SPIRV_OC_OpConvertFToBF16INTEL        : I32EnumAttrCase<"OpConvertFToBF16INTEL", 6116>;
-def SPIRV_OC_OpConvertBF16ToFINTEL        : I32EnumAttrCase<"OpConvertBF16ToFINTEL", 6117>;
+def SPIRV_OC_OpNop                          : I32EnumAttrCase<"OpNop", 0>;
+def SPIRV_OC_OpUndef                        : I32EnumAttrCase<"OpUndef", 1>;
+def SPIRV_OC_OpSourceContinued              : I32EnumAttrCase<"OpSourceContinued", 2>;
+def SPIRV_OC_OpSource                       : I32EnumAttrCase<"OpSource", 3>;
+def SPIRV_OC_OpSourceExtension              : I32EnumAttrCase<"OpSourceExtension", 4>;
+def SPIRV_OC_OpName                         : I32EnumAttrCase<"OpName", 5>;
+def SPIRV_OC_OpMemberName                   : I32EnumAttrCase<"OpMemberName", 6>;
+def SPIRV_OC_OpString                       : I32EnumAttrCase<"OpString", 7>;
+def SPIRV_OC_OpLine                         : I32EnumAttrCase<"OpLine", 8>;
+def SPIRV_OC_OpExtension                    : I32EnumAttrCase<"OpExtension", 10>;
+def SPIRV_OC_OpExtInstImport                : I32EnumAttrCase<"OpExtInstImport", 11>;
+def SPIRV_OC_OpExtInst                      : I32EnumAttrCase<"OpExtInst", 12>;
+def SPIRV_OC_OpMemoryModel                  : I32EnumAttrCase<"OpMemoryModel", 14>;
+def SPIRV_OC_OpEntryPoint                   : I32EnumAttrCase<"OpEntryPoint", 15>;
+def SPIRV_OC_OpExecutionMode                : I32EnumAttrCase<"OpExecutionMode", 16>;
+def SPIRV_OC_OpCapability                   : I32EnumAttrCase<"OpCapability", 17>;
+def SPIRV_OC_OpTypeVoid                     : I32EnumAttrCase<"OpTypeVoid", 19>;
+def SPIRV_OC_OpTypeBool                     : I32EnumAttrCase<"OpTypeBool", 20>;
+def SPIRV_OC_OpTypeInt                      : I32EnumAttrCase<"OpTypeInt", 21>;
+def SPIRV_OC_OpTypeFloat                    : I32EnumAttrCase<"OpTypeFloat", 22>;
+def SPIRV_OC_OpTypeVector                   : I32EnumAttrCase<"OpTypeVector", 23>;
+def SPIRV_OC_OpTypeMatrix                   : I32EnumAttrCase<"OpTypeMatrix", 24>;
+def SPIRV_OC_OpTypeImage                    : I32EnumAttrCase<"OpTypeImage", 25>;
+def SPIRV_OC_OpTypeSampledImage             : I32EnumAttrCase<"OpTypeSampledImage", 27>;
+def SPIRV_OC_OpTypeArray                    : I32EnumAttrCase<"OpTypeArray", 28>;
+def SPIRV_OC_OpTypeRuntimeArray             : I32EnumAttrCase<"OpTypeRuntimeArray", 29>;
+def SPIRV_OC_OpTypeStruct                   : I32EnumAttrCase<"OpTypeStruct", 30>;
+def SPIRV_OC_OpTypePointer                  : I32EnumAttrCase<"OpTypePointer", 32>;
+def SPIRV_OC_OpTypeFunction                 : I32EnumAttrCase<"OpTypeFunction", 33>;
+def SPIRV_OC_OpTypeForwardPointer           : I32EnumAttrCase<"OpTypeForwardPointer", 39>;
+def SPIRV_OC_OpConstantTrue                 : I32EnumAttrCase<"OpConstantTrue", 41>;
+def SPIRV_OC_OpConstantFalse                : I32EnumAttrCase<"OpConstantFalse", 42>;
+def SPIRV_OC_OpConstant                     : I32EnumAttrCase<"OpConstant", 43>;
+def SPIRV_OC_OpConstantComposite            : I32EnumAttrCase<"OpConstantComposite", 44>;
+def SPIRV_OC_OpConstantNull                 : I32EnumAttrCase<"OpConstantNull", 46>;
+def SPIRV_OC_OpSpecConstantTrue             : I32EnumAttrCase<"OpSpecConstantTrue", 48>;
+def SPIRV_OC_OpSpecConstantFalse            : I32EnumAttrCase<"OpSpecConstantFalse", 49>;
+def SPIRV_OC_OpSpecConstant                 : I32EnumAttrCase<"OpSpecConstant", 50>;
+def SPIRV_OC_OpSpecConstantComposite        : I32EnumAttrCase<"OpSpecConstantComposite", 51>;
+def SPIRV_OC_OpSpecConstantOp               : I32EnumAttrCase<"OpSpecConstantOp", 52>;
+def SPIRV_OC_OpFunction                     : I32EnumAttrCase<"OpFunction", 54>;
+def SPIRV_OC_OpFunctionParameter            : I32EnumAttrCase<"OpFunctionParameter", 55>;
+def SPIRV_OC_OpFunctionEnd                  : I32EnumAttrCase<"OpFunctionEnd", 56>;
+def SPIRV_OC_OpFunctionCall                 : I32EnumAttrCase<"OpFunctionCall", 57>;
+def SPIRV_OC_OpVariable                     : I32EnumAttrCase<"OpVariable", 59>;
+def SPIRV_OC_OpLoad                         : I32EnumAttrCase<"OpLoad", 61>;
+def SPIRV_OC_OpStore                        : I32EnumAttrCase<"OpStore", 62>;
+def SPIRV_OC_OpCopyMemory                   : I32EnumAttrCase<"OpCopyMemory", 63>;
+def SPIRV_OC_OpAccessChain                  : I32EnumAttrCase<"OpAccessChain", 65>;
+def SPIRV_OC_OpPtrAccessChain               : I32EnumAttrCase<"OpPtrAccessChain", 67>;
+def SPIRV_OC_OpInBoundsPtrAccessChain       : I32EnumAttrCase<"OpInBoundsPtrAccessChain", 70>;
+def SPIRV_OC_OpDecorate                     : I32EnumAttrCase<"OpDecorate", 71>;
+def SPIRV_OC_OpMemberDecorate               : I32EnumAttrCase<"OpMemberDecorate", 72>;
+def SPIRV_OC_OpVectorExtractDynamic         : I32EnumAttrCase<"OpVectorExtractDynamic", 77>;
+def SPIRV_OC_OpVectorInsertDynamic          : I32EnumAttrCase<"OpVectorInsertDynamic", 78>;
+def SPIRV_OC_OpVectorShuffle                : I32EnumAttrCase<"OpVectorShuffle", 79>;
+def SPIRV_OC_OpCompositeConstruct           : I32EnumAttrCase<"OpCompositeConstruct", 80>;
+def SPIRV_OC_OpCompositeExtract             : I32EnumAttrCase<"OpCompositeExtract", 81>;
+def SPIRV_OC_OpCompositeInsert              : I32EnumAttrCase<"OpCompositeInsert", 82>;
+def SPIRV_OC_OpTranspose                    : I32EnumAttrCase<"OpTranspose", 84>;
+def SPIRV_OC_OpImageDrefGather              : I32EnumAttrCase<"OpImageDrefGather", 97>;
+def SPIRV_OC_OpImage                        : I32EnumAttrCase<"OpImage", 100>;
+def SPIRV_OC_OpImageQuerySize               : I32EnumAttrCase<"OpImageQuerySize", 104>;
+def SPIRV_OC_OpConvertFToU                  : I32EnumAttrCase<"OpConvertFToU", 109>;
+def SPIRV_OC_OpConvertFToS                  : I32EnumAttrCase<"OpConvertFToS", 110>;
+def SPIRV_OC_OpConvertSToF                  : I32EnumAttrCase<"OpConvertSToF", 111>;
+def SPIRV_OC_OpConvertUToF                  : I32EnumAttrCase<"OpConvertUToF", 112>;
+def SPIRV_OC_OpUConvert                     : I32EnumAttrCase<"OpUConvert", 113>;
+def SPIRV_OC_OpSConvert                     : I32EnumAttrCase<"OpSConvert", 114>;
+def SPIRV_OC_OpFConvert                     : I32EnumAttrCase<"OpFConvert", 115>;
+def SPIRV_OC_OpConvertPtrToU                : I32EnumAttrCase<"OpConvertPtrToU", 117>;
+def SPIRV_OC_OpConvertUToPtr                : I32EnumAttrCase<"OpConvertUToPtr", 120>;
+def SPIRV_OC_OpPtrCastToGeneric             : I32EnumAttrCase<"OpPtrCastToGeneric", 121>;
+def SPIRV_OC_OpGenericCastToPtr             : I32EnumAttrCase<"OpGenericCastToPtr", 122>;
+def SPIRV_OC_OpGenericCastToPtrExplicit     : I32EnumAttrCase<"OpGenericCastToPtrExplicit", 123>;
+def SPIRV_OC_OpBitcast                      : I32EnumAttrCase<"OpBitcast", 124>;
+def SPIRV_OC_OpSNegate                      : I32EnumAttrCase<"OpSNegate", 126>;
+def SPIRV_OC_OpFNegate                      : I32EnumAttrCase<"OpFNegate", 127>;
+def SPIRV_OC_OpIAdd                         : I32EnumAttrCase<"OpIAdd", 128>;
+def SPIRV_OC_OpFAdd                         : I32EnumAttrCase<"OpFAdd", 129>;
+def SPIRV_OC_OpISub                         : I32EnumAttrCase<"OpISub", 130>;
+def SPIRV_OC_OpFSub                         : I32EnumAttrCase<"OpFSub", 131>;
+def SPIRV_OC_OpIMul                         : I32EnumAttrCase<"OpIMul", 132>;
+def SPIRV_OC_OpFMul                         : I32EnumAttrCase<"OpFMul", 133>;
+def SPIRV_OC_OpUDiv                         : I32EnumAttrCase<"OpUDiv", 134>;
+def SPIRV_OC_OpSDiv                         : I32EnumAttrCase<"OpSDiv", 135>;
+def SPIRV_OC_OpFDiv                         : I32EnumAttrCase<"OpFDiv", 136>;
+def SPIRV_OC_OpUMod                         : I32EnumAttrCase<"OpUMod", 137>;
+def SPIRV_OC_OpSRem                         : I32EnumAttrCase<"OpSRem", 138>;
+def SPIRV_OC_OpSMod                         : I32EnumAttrCase<"OpSMod", 139>;
+def SPIRV_OC_OpFRem                         : I32EnumAttrCase<"OpFRem", 140>;
+def SPIRV_OC_OpFMod                         : I32EnumAttrCase<"OpFMod", 141>;
+def SPIRV_OC_OpVectorTimesScalar            : I32EnumAttrCase<"OpVectorTimesScalar", 142>;
+def SPIRV_OC_OpMatrixTimesScalar            : I32EnumAttrCase<"OpMatrixTimesScalar", 143>;
+def SPIRV_OC_OpMatrixTimesMatrix            : I32EnumAttrCase<"OpMatrixTimesMatrix", 146>;
+def SPIRV_OC_OpDot                          : I32EnumAttrCase<"OpDot", 148>;
+def SPIRV_OC_OpIAddCarry                    : I32EnumAttrCase<"OpIAddCarry", 149>;
+def SPIRV_OC_OpISubBorrow                   : I32EnumAttrCase<"OpISubBorrow", 150>;
+def SPIRV_OC_OpUMulExtended                 : I32EnumAttrCase<"OpUMulExtended", 151>;
+def SPIRV_OC_OpSMulExtended                 : I32EnumAttrCase<"OpSMulExtended", 152>;
+def SPIRV_OC_OpIsNan                        : I32EnumAttrCase<"OpIsNan", 156>;
+def SPIRV_OC_OpIsInf                        : I32EnumAttrCase<"OpIsInf", 157>;
+def SPIRV_OC_OpOrdered                      : I32EnumAttrCase<"OpOrdered", 162>;
+def SPIRV_OC_OpUnordered                    : I32EnumAttrCase<"OpUnordered", 163>;
+def SPIRV_OC_OpLogicalEqual                 : I32EnumAttrCase<"OpLogicalEqual", 164>;
+def SPIRV_OC_OpLogicalNotEqual              : I32EnumAttrCase<"OpLogicalNotEqual", 165>;
+def SPIRV_OC_OpLogicalOr                    : I32EnumAttrCase<"OpLogicalOr", 166>;
+def SPIRV_OC_OpLogicalAnd                   : I32EnumAttrCase<"OpLogicalAnd", 167>;
+def SPIRV_OC_OpLogicalNot                   : I32EnumAttrCase<"OpLogicalNot", 168>;
+def SPIRV_OC_OpSelect                       : I32EnumAttrCase<"OpSelect", 169>;
+def SPIRV_OC_OpIEqual                       : I32EnumAttrCase<"OpIEqual", 170>;
+def SPIRV_OC_OpINotEqual                    : I32EnumAttrCase<"OpINotEqual", 171>;
+def SPIRV_OC_OpUGreaterThan                 : I32EnumAttrCase<"OpUGreaterThan", 172>;
+def SPIRV_OC_OpSGreaterThan                 : I32EnumAttrCase<"OpSGreaterThan", 173>;
+def SPIRV_OC_OpUGreaterThanEqual            : I32EnumAttrCase<"OpUGreaterThanEqual", 174>;
+def SPIRV_OC_OpSGreaterThanEqual            : I32EnumAttrCase<"OpSGreaterThanEqual", 175>;
+def SPIRV_OC_OpULessThan                    : I32EnumAttrCase<"OpULessThan", 176>;
+def SPIRV_OC_OpSLessThan                    : I32EnumAttrCase<"OpSLessThan", 177>;
+def SPIRV_OC_OpULessThanEqual               : I32EnumAttrCase<"OpULessThanEqual", 178>;
+def SPIRV_OC_OpSLessThanEqual               : I32EnumAttrCase<"OpSLessThanEqual", 179>;
+def SPIRV_OC_OpFOrdEqual                    : I32EnumAttrCase<"OpFOrdEqual", 180>;
+def SPIRV_OC_OpFUnordEqual                  : I32EnumAttrCase<"OpFUnordEqual", 181>;
+def SPIRV_OC_OpFOrdNotEqual                 : I32EnumAttrCase<"OpFOrdNotEqual", 182>;
+def SPIRV_OC_OpFUnordNotEqual               : I32EnumAttrCase<"OpFUnordNotEqual", 183>;
+def SPIRV_OC_OpFOrdLessThan                 : I32EnumAttrCase<"OpFOrdLessThan", 184>;
+def SPIRV_OC_OpFUnordLessThan               : I32EnumAttrCase<"OpFUnordLessThan", 185>;
+def SPIRV_OC_OpFOrdGreaterThan              : I32EnumAttrCase<"OpFOrdGreaterThan", 186>;
+def SPIRV_OC_OpFUnordGreaterThan            : I32EnumAttrCase<"OpFUnordGreaterThan", 187>;
+def SPIRV_OC_OpFOrdLessThanEqual            : I32EnumAttrCase<"OpFOrdLessThanEqual", 188>;
+def SPIRV_OC_OpFUnordLessThanEqual          : I32EnumAttrCase<"OpFUnordLessThanEqual", 189>;
+def SPIRV_OC_OpFOrdGreaterThanEqual         : I32EnumAttrCase<"OpFOrdGreaterThanEqual", 190>;
+def SPIRV_OC_OpFUnordGreaterThanEqual       : I32EnumAttrCase<"OpFUnordGreaterThanEqual", 191>;
+def SPIRV_OC_OpShiftRightLogical            : I32EnumAttrCase<"OpShiftRightLogical", 194>;
+def SPIRV_OC_OpShiftRightArithmetic         : I32EnumAttrCase<"OpShiftRightArithmetic", 195>;
+def SPIRV_OC_OpShiftLeftLogical             : I32EnumAttrCase<"OpShiftLeftLogical", 196>;
+def SPIRV_OC_OpBitwiseOr                    : I32EnumAttrCase<"OpBitwiseOr", 197>;
+def SPIRV_OC_OpBitwiseXor                   : I32EnumAttrCase<"OpBitwiseXor", 198>;
+def SPIRV_OC_OpBitwiseAnd                   : I32EnumAttrCase<"OpBitwiseAnd", 199>;
+def SPIRV_OC_OpNot                          : I32EnumAttrCase<"OpNot", 200>;
+def SPIRV_OC_OpBitFieldInsert               : I32EnumAttrCase<"OpBitFieldInsert", 201>;
+def SPIRV_OC_OpBitFieldSExtract             : I32EnumAttrCase<"OpBitFieldSExtract", 202>;
+def SPIRV_OC_OpBitFieldUExtract             : I32EnumAttrCase<"OpBitFieldUExtract", 203>;
+def SPIRV_OC_OpBitReverse                   : I32EnumAttrCase<"OpBitReverse", 204>;
+def SPIRV_OC_OpBitCount                     : I32EnumAttrCase<"OpBitCount", 205>;
+def SPIRV_OC_OpControlBarrier               : I32EnumAttrCase<"OpControlBarrier", 224>;
+def SPIRV_OC_OpMemoryBarrier                : I32EnumAttrCase<"OpMemoryBarrier", 225>;
+def SPIRV_OC_OpAtomicExchange               : I32EnumAttrCase<"OpAtomicExchange", 229>;
+def SPIRV_OC_OpAtomicCompareExchange        : I32EnumAttrCase<"OpAtomicCompareExchange", 230>;
+def SPIRV_OC_OpAtomicCompareExchangeWeak    : I32EnumAttrCase<"OpAtomicCompareExchangeWeak", 231>;
+def SPIRV_OC_OpAtomicIIncrement             : I32EnumAttrCase<"OpAtomicIIncrement", 232>;
+def SPIRV_OC_OpAtomicIDecrement             : I32EnumAttrCase<"OpAtomicIDecrement", 233>;
+def SPIRV_OC_OpAtomicIAdd                   : I32EnumAttrCase<"OpAtomicIAdd", 234>;
+def SPIRV_OC_OpAtomicISub                   : I32EnumAttrCase<"OpAtomicISub", 235>;
+def SPIRV_OC_OpAtomicSMin                   : I32EnumAttrCase<"OpAtomicSMin", 236>;
+def SPIRV_OC_OpAtomicUMin                   : I32EnumAttrCase<"OpAtomicUMin", 237>;
+def SPIRV_OC_OpAtomicSMax                   : I32EnumAttrCase<"OpAtomicSMax", 238>;
+def SPIRV_OC_OpAtomicUMax                   : I32EnumAttrCase<"OpAtomicUMax", 239>;
+def SPIRV_OC_OpAtomicAnd                    : I32EnumAttrCase<"OpAtomicAnd", 240>;
+def SPIRV_OC_OpAtomicOr                     : I32EnumAttrCase<"OpAtomicOr", 241>;
+def SPIRV_OC_OpAtomicXor                    : I32EnumAttrCase<"OpAtomicXor", 242>;
+def SPIRV_OC_OpPhi                          : I32EnumAttrCase<"OpPhi", 245>;
+def SPIRV_OC_OpLoopMerge                    : I32EnumAttrCase<"OpLoopMerge", 246>;
+def SPIRV_OC_OpSelectionMerge               : I32EnumAttrCase<"OpSelectionMerge", 247>;
+def SPIRV_OC_OpLabel                        : I32EnumAttrCase<"OpLabel", 248>;
+def SPIRV_OC_OpBranch                       : I32EnumAttrCase<"OpBranch", 249>;
+def SPIRV_OC_OpBranchConditional            : I32EnumAttrCase<"OpBranchConditional", 250>;
+def SPIRV_OC_OpReturn                       : I32EnumAttrCase<"OpReturn", 253>;
+def SPIRV_OC_OpReturnValue                  : I32EnumAttrCase<"OpReturnValue", 254>;
+def SPIRV_OC_OpUnreachable                  : I32EnumAttrCase<"OpUnreachable", 255>;
+def SPIRV_OC_OpGroupBroadcast               : I32EnumAttrCase<"OpGroupBroadcast", 263>;
+def SPIRV_OC_OpGroupIAdd                    : I32EnumAttrCase<"OpGroupIAdd", 264>;
+def SPIRV_OC_OpGroupFAdd                    : I32EnumAttrCase<"OpGroupFAdd", 265>;
+def SPIRV_OC_OpGroupFMin                    : I32EnumAttrCase<"OpGroupFMin", 266>;
+def SPIRV_OC_OpGroupUMin                    : I32EnumAttrCase<"OpGroupUMin", 267>;
+def SPIRV_OC_OpGroupSMin                    : I32EnumAttrCase<"OpGroupSMin", 268>;
+def SPIRV_OC_OpGroupFMax                    : I32EnumAttrCase<"OpGroupFMax", 269>;
+def SPIRV_OC_OpGroupUMax                    : I32EnumAttrCase<"OpGroupUMax", 270>;
+def SPIRV_OC_OpGroupSMax                    : I32EnumAttrCase<"OpGroupSMax", 271>;
+def SPIRV_OC_OpNoLine                       : I32EnumAttrCase<"OpNoLine", 317>;
+def SPIRV_OC_OpModuleProcessed              : I32EnumAttrCase<"OpModuleProcessed", 330>;
+def SPIRV_OC_OpGroupNonUniformElect         : I32EnumAttrCase<"OpGroupNonUniformElect", 333>;
+def SPIRV_OC_OpGroupNonUniformBroadcast     : I32EnumAttrCase<"OpGroupNonUniformBroadcast", 337>;
+def SPIRV_OC_OpGroupNonUniformBallot        : I32EnumAttrCase<"OpGroupNonUniformBallot", 339>;
+def SPIRV_OC_OpGroupNonUniformBallotFindLSB : I32EnumAttrCase<"OpGroupNonUniformBallotFindLSB", 343>;
+def SPIRV_OC_OpGroupNonUniformBallotFindMSB : I32EnumAttrCase<"OpGroupNonUniformBallotFindMSB", 344>;
+def SPIRV_OC_OpGroupNonUniformShuffle       : I32EnumAttrCase<"OpGroupNonUniformShuffle", 345>;
+def SPIRV_OC_OpGroupNonUniformShuffleXor    : I32EnumAttrCase<"OpGroupNonUniformShuffleXor", 346>;
+def SPIRV_OC_OpGroupNonUniformShuffleUp     : I32EnumAttrCase<"OpGroupNonUniformShuffleUp", 347>;
+def SPIRV_OC_OpGroupNonUniformShuffleDown   : I32EnumAttrCase<"OpGroupNonUniformShuffleDown", 348>;
+def SPIRV_OC_OpGroupNonUniformIAdd          : I32EnumAttrCase<"OpGroupNonUniformIAdd", 349>;
+def SPIRV_OC_OpGroupNonUniformFAdd          : I32EnumAttrCase<"OpGroupNonUniformFAdd", 350>;
+def SPIRV_OC_OpGroupNonUniformIMul          : I32EnumAttrCase<"OpGroupNonUniformIMul", 351>;
+def SPIRV_OC_OpGroupNonUniformFMul          : I32EnumAttrCase<"OpGroupNonUniformFMul", 352>;
+def SPIRV_OC_OpGroupNonUniformSMin          : I32EnumAttrCase<"OpGroupNonUniformSMin", 353>;
+def SPIRV_OC_OpGroupNonUniformUMin          : I32EnumAttrCase<"OpGroupNonUniformUMin", 354>;
+def SPIRV_OC_OpGroupNonUniformFMin          : I32EnumAttrCase<"OpGroupNonUniformFMin", 355>;
+def SPIRV_OC_OpGroupNonUniformSMax          : I32EnumAttrCase<"OpGroupNonUniformSMax", 356>;
+def SPIRV_OC_OpGroupNonUniformUMax          : I32EnumAttrCase<"OpGroupNonUniformUMax", 357>;
+def SPIRV_OC_OpGroupNonUniformFMax          : I32EnumAttrCase<"OpGroupNonUniformFMax", 358>;
+def SPIRV_OC_OpGroupNonUniformBitwiseAnd    : I32EnumAttrCase<"OpGroupNonUniformBitwiseAnd", 359>;
+def SPIRV_OC_OpGroupNonUniformBitwiseOr     : I32EnumAttrCase<"OpGroupNonUniformBitwiseOr", 360>;
+def SPIRV_OC_OpGroupNonUniformBitwiseXor    : I32EnumAttrCase<"OpGroupNonUniformBitwiseXor", 361>;
+def SPIRV_OC_OpGroupNonUniformLogicalAnd    : I32EnumAttrCase<"OpGroupNonUniformLogicalAnd", 362>;
+def SPIRV_OC_OpGroupNonUniformLogicalOr     : I32EnumAttrCase<"OpGroupNonUniformLogicalOr", 363>;
+def SPIRV_OC_OpGroupNonUniformLogicalXor    : I32EnumAttrCase<"OpGroupNonUniformLogicalXor", 364>;
+def SPIRV_OC_OpSubgroupBallotKHR            : I32EnumAttrCase<"OpSubgroupBallotKHR", 4421>;
+def SPIRV_OC_OpSDot                         : I32EnumAttrCase<"OpSDot", 4450>;
+def SPIRV_OC_OpUDot                         : I32EnumAttrCase<"OpUDot", 4451>;
+def SPIRV_OC_OpSUDot                        : I32EnumAttrCase<"OpSUDot", 4452>;
+def SPIRV_OC_OpSDotAccSat                   : I32EnumAttrCase<"OpSDotAccSat", 4453>;
+def SPIRV_OC_OpUDotAccSat                   : I32EnumAttrCase<"OpUDotAccSat", 4454>;
+def SPIRV_OC_OpSUDotAccSat                  : I32EnumAttrCase<"OpSUDotAccSat", 4455>;
+def SPIRV_OC_OpTypeCooperativeMatrixKHR     : I32EnumAttrCase<"OpTypeCooperativeMatrixKHR", 4456>;
+def SPIRV_OC_OpCooperativeMatrixLoadKHR     : I32EnumAttrCase<"OpCooperativeMatrixLoadKHR", 4457>;
+def SPIRV_OC_OpCooperativeMatrixStoreKHR    : I32EnumAttrCase<"OpCooperativeMatrixStoreKHR", 4458>;
+def SPIRV_OC_OpCooperativeMatrixMulAddKHR   : I32EnumAttrCase<"OpCooperativeMatrixMulAddKHR", 4459>;
+def SPIRV_OC_OpCooperativeMatrixLengthKHR   : I32EnumAttrCase<"OpCooperativeMatrixLengthKHR", 4460>;
+def SPIRV_OC_OpSubgroupBlockReadINTEL       : I32EnumAttrCase<"OpSubgroupBlockReadINTEL", 5575>;
+def SPIRV_OC_OpSubgroupBlockWriteINTEL      : I32EnumAttrCase<"OpSubgroupBlockWriteINTEL", 5576>;
+def SPIRV_OC_OpAssumeTrueKHR                : I32EnumAttrCase<"OpAssumeTrueKHR", 5630>;
+def SPIRV_OC_OpAtomicFAddEXT                : I32EnumAttrCase<"OpAtomicFAddEXT", 6035>;
+def SPIRV_OC_OpConvertFToBF16INTEL          : I32EnumAttrCase<"OpConvertFToBF16INTEL", 6116>;
+def SPIRV_OC_OpConvertBF16ToFINTEL          : I32EnumAttrCase<"OpConvertBF16ToFINTEL", 6117>;
+def SPIRV_OC_OpGroupIMulKHR                 : I32EnumAttrCase<"OpGroupIMulKHR", 6401>;
+def SPIRV_OC_OpGroupFMulKHR                 : I32EnumAttrCase<"OpGroupFMulKHR", 6402>;
 
 def SPIRV_OpcodeAttr :
     SPIRV_I32EnumAttr<"Opcode", "valid SPIR-V instructions", "opcode", [
@@ -4503,10 +4504,10 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpFSub, SPIRV_OC_OpIMul, SPIRV_OC_OpFMul, SPIRV_OC_OpUDiv,
       SPIRV_OC_OpSDiv, SPIRV_OC_OpFDiv, SPIRV_OC_OpUMod, SPIRV_OC_OpSRem,
       SPIRV_OC_OpSMod, SPIRV_OC_OpFRem, SPIRV_OC_OpFMod,
-      SPIRV_OC_OpVectorTimesScalar, SPIRV_OC_OpMatrixTimesScalar, SPIRV_OC_OpDot,
-      SPIRV_OC_OpMatrixTimesMatrix, SPIRV_OC_OpIAddCarry, SPIRV_OC_OpISubBorrow,
-      SPIRV_OC_OpUMulExtended, SPIRV_OC_OpSMulExtended, SPIRV_OC_OpIsNan,
-      SPIRV_OC_OpIsInf, SPIRV_OC_OpOrdered, SPIRV_OC_OpUnordered,
+      SPIRV_OC_OpVectorTimesScalar, SPIRV_OC_OpMatrixTimesScalar,
+      SPIRV_OC_OpMatrixTimesMatrix, SPIRV_OC_OpDot, SPIRV_OC_OpIAddCarry,
+      SPIRV_OC_OpISubBorrow, SPIRV_OC_OpUMulExtended, SPIRV_OC_OpSMulExtended,
+      SPIRV_OC_OpIsNan, SPIRV_OC_OpIsInf, SPIRV_OC_OpOrdered, SPIRV_OC_OpUnordered,
       SPIRV_OC_OpLogicalEqual, SPIRV_OC_OpLogicalNotEqual, SPIRV_OC_OpLogicalOr,
       SPIRV_OC_OpLogicalAnd, SPIRV_OC_OpLogicalNot, SPIRV_OC_OpSelect,
       SPIRV_OC_OpIEqual, SPIRV_OC_OpINotEqual, SPIRV_OC_OpUGreaterThan,
@@ -4536,27 +4537,27 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpGroupFMax, SPIRV_OC_OpGroupUMax, SPIRV_OC_OpGroupSMax,
       SPIRV_OC_OpNoLine, SPIRV_OC_OpModuleProcessed, SPIRV_OC_OpGroupNonUniformElect,
       SPIRV_OC_OpGroupNonUniformBroadcast, SPIRV_OC_OpGroupNonUniformBallot,
-      SPIRV_OC_OpGroupNonUniformShuffle, SPIRV_OC_OpGroupNonUniformShuffleXor,
-      SPIRV_OC_OpGroupNonUniformShuffleUp, SPIRV_OC_OpGroupNonUniformShuffleDown,
-      SPIRV_OC_OpGroupNonUniformIAdd, SPIRV_OC_OpGroupNonUniformFAdd,
-      SPIRV_OC_OpGroupNonUniformIMul, SPIRV_OC_OpGroupNonUniformFMul,
-      SPIRV_OC_OpGroupNonUniformSMin, SPIRV_OC_OpGroupNonUniformUMin,
-      SPIRV_OC_OpGroupNonUniformFMin, SPIRV_OC_OpGroupNonUniformSMax,
-      SPIRV_OC_OpGroupNonUniformUMax, SPIRV_OC_OpGroupNonUniformFMax,
-      SPIRV_OC_OpGroupNonUniformBitwiseAnd, SPIRV_OC_OpGroupNonUniformBitwiseOr,
-      SPIRV_OC_OpGroupNonUniformBitwiseXor, SPIRV_OC_OpGroupNonUniformLogicalAnd,
-      SPIRV_OC_OpGroupNonUniformLogicalOr, SPIRV_OC_OpGroupNonUniformLogicalXor,
-      SPIRV_OC_OpSubgroupBallotKHR, SPIRV_OC_OpSDot, SPIRV_OC_OpUDot,
-      SPIRV_OC_OpSUDot, SPIRV_OC_OpSDotAccSat, SPIRV_OC_OpUDotAccSat,
-      SPIRV_OC_OpSUDotAccSat,
+      SPIRV_OC_OpGroupNonUniformBallotFindLSB,
+      SPIRV_OC_OpGroupNonUniformBallotFindMSB, SPIRV_OC_OpGroupNonUniformShuffle,
+      SPIRV_OC_OpGroupNonUniformShuffleXor, SPIRV_OC_OpGroupNonUniformShuffleUp,
+      SPIRV_OC_OpGroupNonUniformShuffleDown, SPIRV_OC_OpGroupNonUniformIAdd,
+      SPIRV_OC_OpGroupNonUniformFAdd, SPIRV_OC_OpGroupNonUniformIMul,
+      SPIRV_OC_OpGroupNonUniformFMul, SPIRV_OC_OpGroupNonUniformSMin,
+      SPIRV_OC_OpGroupNonUniformUMin, SPIRV_OC_OpGroupNonUniformFMin,
+      SPIRV_OC_OpGroupNonUniformSMax, SPIRV_OC_OpGroupNonUniformUMax,
+      SPIRV_OC_OpGroupNonUniformFMax, SPIRV_OC_OpGroupNonUniformBitwiseAnd,
+      SPIRV_OC_OpGroupNonUniformBitwiseOr, SPIRV_OC_OpGroupNonUniformBitwiseXor,
+      SPIRV_OC_OpGroupNonUniformLogicalAnd, SPIRV_OC_OpGroupNonUniformLogicalOr,
+      SPIRV_OC_OpGroupNonUniformLogicalXor, SPIRV_OC_OpSubgroupBallotKHR,
+      SPIRV_OC_OpSDot, SPIRV_OC_OpUDot, SPIRV_OC_OpSUDot, SPIRV_OC_OpSDotAccSat,
+      SPIRV_OC_OpUDotAccSat, SPIRV_OC_OpSUDotAccSat,
       SPIRV_OC_OpTypeCooperativeMatrixKHR, SPIRV_OC_OpCooperativeMatrixLoadKHR,
       SPIRV_OC_OpCooperativeMatrixStoreKHR, SPIRV_OC_OpCooperativeMatrixMulAddKHR,
-      SPIRV_OC_OpCooperativeMatrixLengthKHR,
-      SPIRV_OC_OpSubgroupBlockReadINTEL, SPIRV_OC_OpSubgroupBlockWriteINTEL,
-      SPIRV_OC_OpAssumeTrueKHR, SPIRV_OC_OpAtomicFAddEXT, SPIRV_OC_OpGroupIMulKHR,
-      SPIRV_OC_OpGroupFMulKHR,
-
-      SPIRV_OC_OpConvertFToBF16INTEL, SPIRV_OC_OpConvertBF16ToFINTEL
+      SPIRV_OC_OpCooperativeMatrixLengthKHR, SPIRV_OC_OpSubgroupBlockReadINTEL,
+      SPIRV_OC_OpSubgroupBlockWriteINTEL, SPIRV_OC_OpAssumeTrueKHR,
+      SPIRV_OC_OpAtomicFAddEXT, SPIRV_OC_OpConvertFToBF16INTEL,
+      SPIRV_OC_OpConvertBF16ToFINTEL, SPIRV_OC_OpGroupIMulKHR,
+      SPIRV_OC_OpGroupFMulKHR
     ]>;
 
 // End opcode section. Generated from SPIR-V spec; DO NOT MODIFY!
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
index e6276e6853fcb0d..a32f625ae821127 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
@@ -56,7 +56,7 @@ def SPIRV_GroupNonUniformBallotOp : SPIRV_Op<"GroupNonUniformBallot", []> {
     #### Example:
 
     ```mlir
-    %0 = spirv.GroupNonUniformBallot "SubGroup" %predicate : vector<4xi32>
+    %0 = spirv.GroupNonUniformBallot <Subgroup> %predicate : vector<4xi32>
     ```
   }];
 
@@ -83,6 +83,118 @@ def SPIRV_GroupNonUniformBallotOp : SPIRV_Op<"GroupNonUniformBallot", []> {
 
 // -----
 
+def SPIRV_GroupNonUniformBallotFindLSBOp : SPIRV_Op<"GroupNonUniformBallotFindLSB", []> {
+  let summary = [{
+    Find the least significant bit set to 1 in Value, considering only the
+    bits in Value required to represent all bits of the group's invocations.
+    If none of the considered bits is set to 1, the resulting value is
+    undefined.
+  }];
+
+  let description = [{
+    Result Type must be a scalar of integer type, whose Signedness operand
+    is 0.
+
+    Execution is a Scope that identifies the group of invocations affected
+    by this command. It must be Subgroup.
+
+    Value must be a vector of four components of integer type scalar, whose
+    Width operand is 32 and whose Signedness operand is 0.
+
+    Value is a set of bitfields where the first invocation is represented in
+    the lowest bit of the first vector component and the last (up to the
+    size of the group) is the higher bit number of the last bitmask needed
+    to represent all bits of the group invocations.
+
+    <!-- End of AutoGen section -->
+
+    #### Example:
+
+    ```mlir
+    %vector = ... : vector<4xi32>
+    %0 = spirv.GroupNonUniformBallotFindLSB <Subgroup> %vector : vector<4xi32>, i32
+    ```
+  }];
+
+  let availability = [
+    MinVersion<SPIRV_V_1_3>,
+    MaxVersion<SPIRV_V_1_6>,
+    Extension<[]>,
+    Capability<[SPIRV_C_GroupNonUniformBallot]>
+  ];
+
+  let arguments = (ins
+    SPIRV_ScopeAttr:$execution_scope,
+    SPIRV_IOrUIVec4:$value
+  );
+
+  let results = (outs
+    SPIRV_SignlessOrUnsignedInt:$result
+  );
+
+  let assemblyFormat = [{
+    $execution_scope $value attr-dict `:` type($value) `,` type($result)
+  }];
+}
+
+// -----
+
+def SPIRV_GroupNonUniformBallotFindMSBOp : SPIRV_Op<"GroupNonUniformBallotFindMSB", []> {
+  let summary = [{
+    Find the most significant bit set to 1 in Value, considering only the
+    bits in Value required to represent all bits of the group's invocations.
+    If none of the considered bits is set to 1, the resulting value is
+    undefined.
+  }];
+
+  let description = [{
+    Result Type must be a scalar of integer type, whose Signedness operand
+    is 0.
+
+    Execution is a Scope that identifies the group of invocations affected
+    by this command. It must be Subgroup.
+
+    Value must be a vector of four components of integer type scalar, whose
+    Width operand is 32 and whose Signedness operand is 0.
+
+    Value is a set of bitfields where the first invocation is represented in
+    the lowest bit of the first vector component and the last (up to the
+    size of the group) is the higher bit number of the last bitmask needed
+    to represent all bits of the group invocations.
+
+    <!-- End of AutoGen section -->
+
+    #### Example:
+
+    ```mlir
+    %vector = ... : vector<4xi32>
+    %0 = spirv.GroupNonUniformBallotFindMSB <Subgroup> %vector : vector<4xi32>, i32
+    ```
+  }];
+
+  let availability = [
+    MinVersion<SPIRV_V_1_3>,
+    MaxVersion<SPIRV_V_1_6>,
+    Extension<[]>,
+    Capability<[SPIRV_C_GroupNonUniformBallot]>
+  ];
+
+  let arguments = (ins
+    SPIRV_ScopeAttr:$execution_scope,
+    SPIRV_IOrUIVec4:$value
+  );
+
+  let results = (outs
+    SPIRV_SignlessOrUnsignedInt:$result
+  );
+
+  let assemblyFormat = [{
+    $execution_scope $value attr-dict `:` type($value) `,` type($result)
+  }];
+}
+
+// -----
+
 def SPIRV_GroupNonUniformBroadcastOp : SPIRV_Op<"GroupNonUniformBroadcast",
   [Pure, AllTypesMatch<["value", "result"]>]> {
   let summary = [{
diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
index 820a18731ffdb03..aad2eab83dbd383 100644
--- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
+++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
@@ -410,7 +410,7 @@ def ApplyFoldElementwiseToVectorPatternsOp : Op<Transform_Dialect,
     "apply_patterns.vector.elementwise_to_vector",
     [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
   let description = [{
-    Collect a set of patterns that fold elementwise op on vectors to the vector 
+    Collect a set of patterns that fold elementwise op on vectors to the vector
     dialect.
   }];
 
@@ -426,8 +426,9 @@ def ApplyVectorReductionToContractPatternsOp : Op<Transform_Dialect,
       - CombineContractBroadcast
       - CombineContractABTranspose
       - CombineContractResultTranspose
-      - ReorderCastOpsOnBroadcast
       - ReorderElementwiseOpsOnTranspose
+      - ReorderElementwiseOpsOnBroadcast
+      - ReorderCastOpsOnBroadcast
 
     These patterns have the effect of rewriting a vector.multi_reduce into a
     vector.contract.
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
index 10970fd03e6eb2e..456d021bc793581 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
@@ -144,9 +144,22 @@ void populateVectorTransferFullPartialPatterns(
 void populateVectorTransferCollapseInnerMostContiguousDimsPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit = 1);
 
-/// Patterns that remove redundant vector broadcasts.
-void populateSinkVectorBroadcastPatterns(RewritePatternSet &patterns,
-                                         PatternBenefit benefit = 1);
+/// Patterns that remove redundant Vector Ops by re-ordering them with
+/// e.g. elementwise Ops:
+/// ```
+/// %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+/// %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+/// %r = arith.addf %at, %bt : vector<2x4xf32>
+/// ```
+/// gets converted to:
+/// ```
+/// %0 = arith.addf %a, %b : vector<4x2xf32>
+/// %r = vector.transpose %0, [1, 0] : vector<2x4xf32>
+/// ```
+/// At the moment, these patterns are limited to vector.broadcast and
+/// vector.transpose.
+void populateSinkVectorOpsPatterns(RewritePatternSet &patterns,
+                                   PatternBenefit benefit = 1);
 
 /// Patterns that fold chained vector reductions. These patterns assume that
 /// elementwise operations (e.g., `arith.addf` with vector operands) are
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index b808738804030f8..809e9448e80abfd 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -844,6 +844,155 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
   return success();
 }
 
+// Implement the AMDGPU_DPPLowering class that will convert the amdgpu.dpp
+// operation into the corresponding ROCDL instructions.
+struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
+  AMDGPUDPPLowering(LLVMTypeConverter &converter, Chipset chipset)
+      : ConvertOpToLLVMPattern<DPPOp>(converter), chipset(chipset) {}
+  Chipset chipset;
+
+  LogicalResult
+  matchAndRewrite(DPPOp DppOp, DPPOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    // Convert the source operand to the corresponding LLVM type
+    Location loc = DppOp.getLoc();
+    Value src = adaptor.getSrc();
+    Value old = adaptor.getOld();
+    Type srcType = src.getType();
+    Type oldType = old.getType();
+    Type llvmType = nullptr;
+    if (srcType.getIntOrFloatBitWidth() < 32) {
+      llvmType = rewriter.getI32Type();
+    } else if (isa<FloatType>(srcType)) {
+      llvmType = (srcType.getIntOrFloatBitWidth() == 32)
+                     ? rewriter.getF32Type()
+                     : rewriter.getF64Type();
+    } else if (isa<IntegerType>(srcType)) {
+      llvmType = (srcType.getIntOrFloatBitWidth() == 32)
+                     ? rewriter.getI32Type()
+                     : rewriter.getI64Type();
+    }
+    auto llvmSrcIntType = typeConverter->convertType(
+        rewriter.getIntegerType(srcType.getIntOrFloatBitWidth()));
+
+    // If the source type is less of 32, use bitcast to convert it to i32.
+    auto convertOperand = [&](Value operand, Type operandType) {
+      if (operandType.getIntOrFloatBitWidth() <= 16) {
+        if (llvm::isa<FloatType>(operandType)) {
+          operand =
+              rewriter.create<LLVM::BitcastOp>(loc, llvmSrcIntType, operand);
+        }
+        auto llvmVecType = typeConverter->convertType(mlir::VectorType::get(
+            32 / operandType.getIntOrFloatBitWidth(), llvmSrcIntType));
+        Value undefVec = rewriter.create<LLVM::UndefOp>(loc, llvmVecType);
+        operand = rewriter.create<LLVM::InsertElementOp>(
+            loc, undefVec, operand, createI32Constant(rewriter, loc, 0));
+        operand = rewriter.create<LLVM::BitcastOp>(loc, llvmType, operand);
+      }
+      return operand;
+    };
+
+    src = convertOperand(src, srcType);
+    old = convertOperand(old, oldType);
+
+    // This is taken from the following file llvm/lib/Target/AMDGPU/SIDefines.h
+    enum DppCtrl : unsigned {
+      ROW_SHL0 = 0x100,
+      ROW_SHR0 = 0x110,
+      ROW_ROR0 = 0x120,
+      WAVE_SHL1 = 0x130,
+      WAVE_ROL1 = 0x134,
+      WAVE_SHR1 = 0x138,
+      WAVE_ROR1 = 0x13C,
+      ROW_MIRROR = 0x140,
+      ROW_HALF_MIRROR = 0x141,
+      BCAST15 = 0x142,
+      BCAST31 = 0x143,
+    };
+
+    auto kind = DppOp.getKind();
+    auto permArgument = DppOp.getPermArgument();
+    uint32_t DppCtrl = 0;
+
+    switch (kind) {
+
+    case DPPPerm::quad_perm:
+      if (auto quadPermAttr = cast<ArrayAttr>(*permArgument)) {
+        int32_t i = 0;
+        for (auto elem : quadPermAttr.getAsRange<IntegerAttr>()) {
+          uint32_t num = elem.getInt();
+          DppCtrl |= num << (i * 2);
+          i++;
+        }
+      }
+      break;
+    case DPPPerm::row_shl:
+      if (auto intAttr = cast<IntegerAttr>(*permArgument)) {
+        DppCtrl = intAttr.getInt() + DppCtrl::ROW_SHL0;
+      }
+      break;
+    case DPPPerm::row_shr:
+      if (auto intAttr = cast<IntegerAttr>(*permArgument)) {
+        DppCtrl = intAttr.getInt() + DppCtrl::ROW_SHR0;
+      }
+      break;
+    case DPPPerm::row_ror:
+      if (auto intAttr = cast<IntegerAttr>(*permArgument)) {
+        DppCtrl = intAttr.getInt() + DppCtrl::ROW_ROR0;
+      }
+      break;
+    case DPPPerm::wave_shl:
+      DppCtrl = DppCtrl::WAVE_SHL1;
+      break;
+    case DPPPerm::wave_shr:
+      DppCtrl = DppCtrl::WAVE_SHR1;
+      break;
+    case DPPPerm::wave_rol:
+      DppCtrl = DppCtrl::WAVE_ROL1;
+      break;
+    case DPPPerm::wave_ror:
+      DppCtrl = DppCtrl::WAVE_ROR1;
+      break;
+    case DPPPerm::row_mirror:
+      DppCtrl = DppCtrl::ROW_MIRROR;
+      break;
+    case DPPPerm::row_half_mirror:
+      DppCtrl = DppCtrl::ROW_HALF_MIRROR;
+      break;
+    case DPPPerm::row_bcast_15:
+      DppCtrl = DppCtrl::BCAST15;
+      break;
+    case DPPPerm::row_bcast_31:
+      DppCtrl = DppCtrl::BCAST31;
+      break;
+    }
+
+    // Check for row_mask, bank_mask, bound_ctrl if they exist and create
+    // constants
+    auto rowMask = DppOp->getAttrOfType<IntegerAttr>("row_mask").getInt();
+    auto bankMask = DppOp->getAttrOfType<IntegerAttr>("bank_mask").getInt();
+    bool boundCtrl = DppOp->getAttrOfType<BoolAttr>("bound_ctrl").getValue();
+
+    // create a ROCDL_DPPMovOp instruction with the appropriate attributes
+    auto dppMovOp = rewriter.create<ROCDL::DPPUpdateOp>(
+        loc, llvmType, old, src, DppCtrl, rowMask, bankMask, boundCtrl);
+
+    Value result = dppMovOp.getRes();
+    if (srcType.getIntOrFloatBitWidth() < 32) {
+      result = rewriter.create<LLVM::TruncOp>(loc, llvmSrcIntType, result);
+      if (!llvm::isa<IntegerType>(srcType)) {
+        result = rewriter.create<LLVM::BitcastOp>(loc, srcType, result);
+      }
+    }
+
+    // We are replacing the AMDGPU_DPPOp instruction with the new
+    // ROCDL_DPPMovOp instruction
+    rewriter.replaceOp(DppOp, ValueRange(result));
+    return success();
+  }
+};
+
 struct ConvertAMDGPUToROCDLPass
     : public impl::ConvertAMDGPUToROCDLBase<ConvertAMDGPUToROCDLPass> {
   ConvertAMDGPUToROCDLPass() = default;
@@ -895,9 +1044,10 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
                                ROCDL::RawPtrBufferAtomicUminOp>,
            RawBufferOpLowering<RawBufferAtomicCmpswapOp,
                                ROCDL::RawPtrBufferAtomicCmpSwap>,
-           LDSBarrierOpLowering, SchedBarrierOpLowering, MFMAOpLowering,
-           WMMAOpLowering, ExtPackedFp8OpLowering, PackedTrunc2xFp8OpLowering,
-           PackedStochRoundFp8OpLowering>(converter, chipset);
+           AMDGPUDPPLowering, LDSBarrierOpLowering, SchedBarrierOpLowering,
+           MFMAOpLowering, WMMAOpLowering, ExtPackedFp8OpLowering,
+           PackedTrunc2xFp8OpLowering, PackedStochRoundFp8OpLowering>(converter,
+                                                                      chipset);
 }
 
 std::unique_ptr<Pass> mlir::createConvertAMDGPUToROCDLPass() {
diff --git a/mlir/lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp b/mlir/lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp
new file mode 100644
index 000000000000000..fcabad32974acb7
--- /dev/null
+++ b/mlir/lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp
@@ -0,0 +1,23 @@
+//===- AttrToSPIRVConverter.cpp - GPU attributes conversion to SPIR-V - C++===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h>
+
+namespace mlir {
+spirv::StorageClass addressSpaceToStorageClass(gpu::AddressSpace addressSpace) {
+  switch (addressSpace) {
+  case gpu::AddressSpace::Global:
+    return spirv::StorageClass::CrossWorkgroup;
+  case gpu::AddressSpace::Workgroup:
+    return spirv::StorageClass::Workgroup;
+  case gpu::AddressSpace::Private:
+    return spirv::StorageClass::Private;
+  }
+  llvm_unreachable("Unhandled storage class");
+}
+} // namespace mlir
diff --git a/mlir/lib/Conversion/GPUCommon/CMakeLists.txt b/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
index 55bd168b83a0ad4..ce914c0ea3dd8aa 100644
--- a/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
@@ -15,6 +15,7 @@ if (MLIR_ENABLE_ROCM_CONVERSIONS)
 endif()
 
 add_mlir_conversion_library(MLIRGPUToGPURuntimeTransforms
+  AttrToSPIRVConverter.cpp
   GPUToLLVMConversion.cpp
   GPUOpsLowering.cpp
 
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index 36e4a6a38a68e47..ced4236402923ad 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -9,6 +9,8 @@
 #include "mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h"
 
 #include "../GPUCommon/GPUOpsLowering.h"
+#include "mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h"
+#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
@@ -328,6 +330,7 @@ struct GPUToLLVMSPVConversionPass final
                         gpu::ReturnOp, gpu::ShuffleOp, gpu::ThreadIdOp>();
 
     populateGpuToLLVMSPVConversionPatterns(converter, patterns);
+    populateGpuMemorySpaceAttributeConversions(converter);
 
     if (failed(applyPartialConversion(getOperation(), target,
                                       std::move(patterns))))
@@ -341,6 +344,15 @@ struct GPUToLLVMSPVConversionPass final
 //===----------------------------------------------------------------------===//
 
 namespace mlir {
+namespace {
+static unsigned
+gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace addressSpace) {
+  constexpr spirv::ClientAPI clientAPI = spirv::ClientAPI::OpenCL;
+  return storageClassToAddressSpace(clientAPI,
+                                    addressSpaceToStorageClass(addressSpace));
+}
+} // namespace
+
 void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &typeConverter,
                                             RewritePatternSet &patterns) {
   patterns.add<GPUBarrierConversion, GPUReturnOpLowering, GPUShuffleConversion,
@@ -349,12 +361,11 @@ void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &typeConverter,
                LaunchConfigOpConversion<gpu::BlockDimOp>,
                LaunchConfigOpConversion<gpu::ThreadIdOp>,
                LaunchConfigOpConversion<gpu::GlobalIdOp>>(typeConverter);
-  constexpr spirv::ClientAPI clientAPI = spirv::ClientAPI::OpenCL;
   MLIRContext *context = &typeConverter.getContext();
   unsigned privateAddressSpace =
-      storageClassToAddressSpace(clientAPI, spirv::StorageClass::Function);
+      gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace::Private);
   unsigned localAddressSpace =
-      storageClassToAddressSpace(clientAPI, spirv::StorageClass::Workgroup);
+      gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace::Workgroup);
   OperationName llvmFuncOpName(LLVM::LLVMFuncOp::getOperationName(), context);
   StringAttr kernelBlockSizeAttributeName =
       LLVM::LLVMFuncOp::getReqdWorkGroupSizeAttrName(llvmFuncOpName);
@@ -366,4 +377,9 @@ void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &typeConverter,
           LLVM::CConv::SPIR_KERNEL, LLVM::CConv::SPIR_FUNC,
           /*encodeWorkgroupAttributionsAsArguments=*/true});
 }
+
+void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter) {
+  populateGpuMemorySpaceAttributeConversions(typeConverter,
+                                             gpuAddressSpaceToOCLAddressSpace);
+}
 } // namespace mlir
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 5b3c2fb15e70263..52ff138bedf65bd 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -291,6 +291,65 @@ struct Log1pOpPattern final : public OpConversionPattern<math::Log1pOp> {
   }
 };
 
+/// Converts math.log2 and math.log10 to SPIR-V ops.
+///
+/// SPIR-V does not have direct operations for log2 and log10. Explicitly
+/// lower to these operations using:
+///   log2(x) = log(x) * 1/log(2)
+///   log10(x) = log(x) * 1/log(10)
+
+template <typename MathLogOp, typename SpirvLogOp>
+struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
+  using OpConversionPattern<MathLogOp>::OpConversionPattern;
+  using typename OpConversionPattern<MathLogOp>::OpAdaptor;
+
+  static constexpr double log2Reciprocal =
+      1.442695040888963407359924681001892137426645954152985934135449407;
+  static constexpr double log10Reciprocal =
+      0.4342944819032518276511289189166050822943970058036665661144537832;
+
+  LogicalResult
+  matchAndRewrite(MathLogOp operation, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    assert(adaptor.getOperands().size() == 1);
+    if (LogicalResult res = checkSourceOpTypes(rewriter, operation);
+        failed(res))
+      return res;
+
+    Location loc = operation.getLoc();
+    Type type = this->getTypeConverter()->convertType(operation.getType());
+    if (!type)
+      return rewriter.notifyMatchFailure(operation, "type conversion failed");
+
+    auto getConstantValue = [&](double value) {
+      if (auto floatType = dyn_cast<FloatType>(type)) {
+        return rewriter.create<spirv::ConstantOp>(
+            loc, type, rewriter.getFloatAttr(floatType, value));
+      }
+      if (auto vectorType = dyn_cast<VectorType>(type)) {
+        Type elemType = vectorType.getElementType();
+
+        if (isa<FloatType>(elemType)) {
+          return rewriter.create<spirv::ConstantOp>(
+              loc, type,
+              DenseFPElementsAttr::get(
+                  vectorType, FloatAttr::get(elemType, value).getValue()));
+        }
+      }
+
+      llvm_unreachable("unimplemented types for log2/log10");
+    };
+
+    Value constantValue = getConstantValue(
+        std::is_same<MathLogOp, math::Log2Op>() ? log2Reciprocal
+                                                : log10Reciprocal);
+    Value log = rewriter.create<SpirvLogOp>(loc, adaptor.getOperand());
+    rewriter.replaceOpWithNewOp<spirv::FMulOp>(operation, type, log,
+                                               constantValue);
+    return success();
+  }
+};
+
 /// Converts math.powf to SPIRV-Ops.
 struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
   using OpConversionPattern::OpConversionPattern;
@@ -411,6 +470,8 @@ void populateMathToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
   // GLSL patterns
   patterns
       .add<CountLeadingZerosPattern, Log1pOpPattern<spirv::GLLogOp>,
+           Log2Log10OpPattern<math::Log2Op, spirv::GLLogOp>,
+           Log2Log10OpPattern<math::Log10Op, spirv::GLLogOp>,
            ExpM1OpPattern<spirv::GLExpOp>, PowFOpPattern, RoundOpPattern,
            CheckedElementwiseOpPattern<math::AbsFOp, spirv::GLFAbsOp>,
            CheckedElementwiseOpPattern<math::AbsIOp, spirv::GLSAbsOp>,
@@ -430,6 +491,8 @@ void populateMathToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
 
   // OpenCL patterns
   patterns.add<Log1pOpPattern<spirv::CLLogOp>, ExpM1OpPattern<spirv::CLExpOp>,
+               Log2Log10OpPattern<math::Log2Op, spirv::CLLogOp>,
+               Log2Log10OpPattern<math::Log10Op, spirv::CLLogOp>,
                CheckedElementwiseOpPattern<math::AbsFOp, spirv::CLFAbsOp>,
                CheckedElementwiseOpPattern<math::AbsIOp, spirv::CLSAbsOp>,
                CheckedElementwiseOpPattern<math::AtanOp, spirv::CLAtanOp>,
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index 171a9b7c92dc55e..15c96589705748e 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -100,6 +100,19 @@ static void lowerYield(SmallVector<Value> &resultVariables,
   rewriter.eraseOp(yield);
 }
 
+// Lower the contents of an scf::if/scf::index_switch regions to an
+// emitc::if/emitc::switch region. The contents of the lowering region is
+// moved into the respective lowered region, but the scf::yield is replaced not
+// only with an emitc::yield, but also with a sequence of emitc::assign ops that
+// set the yielded values into the result variables.
+static void lowerRegion(SmallVector<Value> &resultVariables,
+                        ConversionPatternRewriter &rewriter, Region &region,
+                        Region &loweredRegion) {
+  rewriter.inlineRegionBefore(region, loweredRegion, loweredRegion.end());
+  Operation *terminator = loweredRegion.back().getTerminator();
+  lowerYield(resultVariables, rewriter, cast<scf::YieldOp>(terminator));
+}
+
 LogicalResult
 ForLowering::matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
                              ConversionPatternRewriter &rewriter) const {
@@ -161,18 +174,6 @@ IfLowering::matchAndRewrite(IfOp ifOp, OpAdaptor adaptor,
     return rewriter.notifyMatchFailure(ifOp,
                                        "create variables for results failed");
 
-  // Utility function to lower the contents of an scf::if region to an emitc::if
-  // region. The contents of the scf::if regions is moved into the respective
-  // emitc::if regions, but the scf::yield is replaced not only with an
-  // emitc::yield, but also with a sequence of emitc::assign ops that set the
-  // yielded values into the result variables.
-  auto lowerRegion = [&resultVariables, &rewriter](Region &region,
-                                                   Region &loweredRegion) {
-    rewriter.inlineRegionBefore(region, loweredRegion, loweredRegion.end());
-    Operation *terminator = loweredRegion.back().getTerminator();
-    lowerYield(resultVariables, rewriter, cast<scf::YieldOp>(terminator));
-  };
-
   Region &thenRegion = adaptor.getThenRegion();
   Region &elseRegion = adaptor.getElseRegion();
 
@@ -182,21 +183,64 @@ IfLowering::matchAndRewrite(IfOp ifOp, OpAdaptor adaptor,
       rewriter.create<emitc::IfOp>(loc, adaptor.getCondition(), false, false);
 
   Region &loweredThenRegion = loweredIf.getThenRegion();
-  lowerRegion(thenRegion, loweredThenRegion);
+  lowerRegion(resultVariables, rewriter, thenRegion, loweredThenRegion);
 
   if (hasElseBlock) {
     Region &loweredElseRegion = loweredIf.getElseRegion();
-    lowerRegion(elseRegion, loweredElseRegion);
+    lowerRegion(resultVariables, rewriter, elseRegion, loweredElseRegion);
   }
 
   rewriter.replaceOp(ifOp, resultVariables);
   return success();
 }
 
+// Lower scf::index_switch to emitc::switch, implementing result values as
+// emitc::variable's updated within the case and default regions.
+struct IndexSwitchOpLowering : public OpConversionPattern<IndexSwitchOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(IndexSwitchOp indexSwitchOp, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
+LogicalResult IndexSwitchOpLowering::matchAndRewrite(
+    IndexSwitchOp indexSwitchOp, OpAdaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  Location loc = indexSwitchOp.getLoc();
+
+  // Create an emitc::variable op for each result. These variables will be
+  // assigned to by emitc::assign ops within the case and default regions.
+  SmallVector<Value> resultVariables;
+  if (failed(createVariablesForResults(indexSwitchOp, getTypeConverter(),
+                                       rewriter, resultVariables))) {
+    return rewriter.notifyMatchFailure(indexSwitchOp,
+                                       "create variables for results failed");
+  }
+
+  auto loweredSwitch = rewriter.create<emitc::SwitchOp>(
+      loc, adaptor.getArg(), adaptor.getCases(), indexSwitchOp.getNumCases());
+
+  // Lowering all case regions.
+  for (auto pair :
+       llvm::zip(adaptor.getCaseRegions(), loweredSwitch.getCaseRegions())) {
+    lowerRegion(resultVariables, rewriter, *std::get<0>(pair),
+                std::get<1>(pair));
+  }
+
+  // Lowering default region.
+  lowerRegion(resultVariables, rewriter, adaptor.getDefaultRegion(),
+              loweredSwitch.getDefaultRegion());
+
+  rewriter.replaceOp(indexSwitchOp, resultVariables);
+  return success();
+}
+
 void mlir::populateSCFToEmitCConversionPatterns(RewritePatternSet &patterns,
                                                 TypeConverter &typeConverter) {
   patterns.add<ForLowering>(typeConverter, patterns.getContext());
   patterns.add<IfLowering>(typeConverter, patterns.getContext());
+  patterns.add<IndexSwitchOpLowering>(typeConverter, patterns.getContext());
 }
 
 void SCFToEmitCPass::runOnOperation() {
@@ -211,7 +255,7 @@ void SCFToEmitCPass::runOnOperation() {
 
   // Configure conversion to lower out SCF operations.
   ConversionTarget target(getContext());
-  target.addIllegalOp<scf::ForOp, scf::IfOp>();
+  target.addIllegalOp<scf::ForOp, scf::IfOp, scf::IndexSwitchOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index e3beceaa3bbb5bf..c1a785fb25478da 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -326,6 +326,68 @@ LogicalResult MFMAOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// DPPOp
+//===----------------------------------------------------------------------===//
+LogicalResult DPPOp::verify() {
+  Type srcType = getSrc().getType();
+  if (srcType.getIntOrFloatBitWidth() > 64) {
+    return emitOpError("integer and floating point types larger than 64 bits "
+                       "are not supported");
+  }
+
+  DPPPerm kind = getKind();
+  Attribute permArgument = getPermArgument().value_or(Attribute{});
+
+  switch (kind) {
+
+  case DPPPerm::quad_perm: {
+    auto quadPermAttr = dyn_cast_or_null<ArrayAttr>(permArgument);
+    if (!quadPermAttr || quadPermAttr.size() != 4) {
+      return emitOpError("quad_perm attribute must have exactly 4 elements");
+    }
+    for (auto elem : quadPermAttr.getAsRange<IntegerAttr>()) {
+      uint32_t num = elem.getInt();
+      if (num < 0 || num > 3) {
+        return emitOpError(
+            "Each element of quad_perm must be in the range [0, 3]");
+      }
+    }
+  } break;
+
+  case DPPPerm::row_shl:
+  case DPPPerm::row_shr:
+  case DPPPerm::row_ror: {
+    if (!permArgument) {
+      return emitOpError("Attribute '" + Twine(stringifyDPPPerm(kind)) +
+                         "' value not specified");
+    }
+    if (auto intAttr = dyn_cast<IntegerAttr>(permArgument)) {
+      uint32_t attrValue = intAttr.getInt();
+      if (attrValue < 1 || attrValue > 15) {
+        return emitOpError("Attribute value must be between 1 and 15");
+      }
+    }
+  } break;
+
+  case DPPPerm::wave_shl:
+  case DPPPerm::wave_shr:
+  case DPPPerm::wave_rol:
+  case DPPPerm::wave_ror:
+  case DPPPerm::row_mirror:
+  case DPPPerm::row_half_mirror:
+  case DPPPerm::row_bcast_15:
+  case DPPPerm::row_bcast_31: {
+    if (permArgument && !isa<UnitAttr>(permArgument)) {
+      return emitOpError("Expected unit attribute for permArgument, but found "
+                         "non-trivial argument");
+    }
+    break;
+  }
+  }
+  return success();
+}
+
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUEnums.cpp.inc"
 
 #define GET_ATTRDEF_CLASSES
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index 79176386d5c2c24..0dfb7ae18eb06a0 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -1087,6 +1087,138 @@ GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// SwitchOp
+//===----------------------------------------------------------------------===//
+
+/// Parse the case regions and values.
+static ParseResult
+parseSwitchCases(OpAsmParser &parser, DenseI64ArrayAttr &cases,
+                 SmallVectorImpl<std::unique_ptr<Region>> &caseRegions) {
+  SmallVector<int64_t> caseValues;
+  while (succeeded(parser.parseOptionalKeyword("case"))) {
+    int64_t value;
+    Region &region = *caseRegions.emplace_back(std::make_unique<Region>());
+    if (parser.parseInteger(value) ||
+        parser.parseRegion(region, /*arguments=*/{}))
+      return failure();
+    caseValues.push_back(value);
+  }
+  cases = parser.getBuilder().getDenseI64ArrayAttr(caseValues);
+  return success();
+}
+
+/// Print the case regions and values.
+static void printSwitchCases(OpAsmPrinter &p, Operation *op,
+                             DenseI64ArrayAttr cases, RegionRange caseRegions) {
+  for (auto [value, region] : llvm::zip(cases.asArrayRef(), caseRegions)) {
+    p.printNewline();
+    p << "case " << value << ' ';
+    p.printRegion(*region, /*printEntryBlockArgs=*/false);
+  }
+}
+
+static LogicalResult verifyRegion(emitc::SwitchOp op, Region &region,
+                                  const Twine &name) {
+  auto yield = dyn_cast<emitc::YieldOp>(region.front().back());
+  if (!yield)
+    return op.emitOpError("expected region to end with emitc.yield, but got ")
+           << region.front().back().getName();
+
+  if (yield.getNumOperands() != 0) {
+    return (op.emitOpError("expected each region to return ")
+            << "0 values, but " << name << " returns "
+            << yield.getNumOperands())
+               .attachNote(yield.getLoc())
+           << "see yield operation here";
+  }
+
+  return success();
+}
+
+LogicalResult emitc::SwitchOp::verify() {
+  if (!isIntegerIndexOrOpaqueType(getArg().getType()))
+    return emitOpError("unsupported type ") << getArg().getType();
+
+  if (getCases().size() != getCaseRegions().size()) {
+    return emitOpError("has ")
+           << getCaseRegions().size() << " case regions but "
+           << getCases().size() << " case values";
+  }
+
+  DenseSet<int64_t> valueSet;
+  for (int64_t value : getCases())
+    if (!valueSet.insert(value).second)
+      return emitOpError("has duplicate case value: ") << value;
+
+  if (failed(verifyRegion(*this, getDefaultRegion(), "default region")))
+    return failure();
+
+  for (auto [idx, caseRegion] : llvm::enumerate(getCaseRegions()))
+    if (failed(verifyRegion(*this, caseRegion, "case region #" + Twine(idx))))
+      return failure();
+
+  return success();
+}
+
+unsigned emitc::SwitchOp::getNumCases() { return getCases().size(); }
+
+Block &emitc::SwitchOp::getDefaultBlock() { return getDefaultRegion().front(); }
+
+Block &emitc::SwitchOp::getCaseBlock(unsigned idx) {
+  assert(idx < getNumCases() && "case index out-of-bounds");
+  return getCaseRegions()[idx].front();
+}
+
+void SwitchOp::getSuccessorRegions(
+    RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &successors) {
+  llvm::copy(getRegions(), std::back_inserter(successors));
+}
+
+void SwitchOp::getEntrySuccessorRegions(
+    ArrayRef<Attribute> operands,
+    SmallVectorImpl<RegionSuccessor> &successors) {
+  FoldAdaptor adaptor(operands, *this);
+
+  // If a constant was not provided, all regions are possible successors.
+  auto arg = dyn_cast_or_null<IntegerAttr>(adaptor.getArg());
+  if (!arg) {
+    llvm::copy(getRegions(), std::back_inserter(successors));
+    return;
+  }
+
+  // Otherwise, try to find a case with a matching value. If not, the
+  // default region is the only successor.
+  for (auto [caseValue, caseRegion] : llvm::zip(getCases(), getCaseRegions())) {
+    if (caseValue == arg.getInt()) {
+      successors.emplace_back(&caseRegion);
+      return;
+    }
+  }
+  successors.emplace_back(&getDefaultRegion());
+}
+
+void SwitchOp::getRegionInvocationBounds(
+    ArrayRef<Attribute> operands, SmallVectorImpl<InvocationBounds> &bounds) {
+  auto operandValue = llvm::dyn_cast_or_null<IntegerAttr>(operands.front());
+  if (!operandValue) {
+    // All regions are invoked at most once.
+    bounds.append(getNumRegions(), InvocationBounds(/*lb=*/0, /*ub=*/1));
+    return;
+  }
+
+  unsigned liveIndex = getNumRegions() - 1;
+  const auto *iteratorToInt = llvm::find(getCases(), operandValue.getInt());
+
+  liveIndex = iteratorToInt != getCases().end()
+                  ? std::distance(getCases().begin(), iteratorToInt)
+                  : liveIndex;
+
+  for (unsigned regIndex = 0, regNum = getNumRegions(); regIndex < regNum;
+       ++regIndex)
+    bounds.emplace_back(/*lb=*/0, /*ub=*/regIndex == liveIndex);
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index 504f63b48c9433d..1399d419735db95 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -271,37 +271,41 @@ getUnderlyingObjectSet(Value pointerValue) {
 static void createNewAliasScopesFromNoAliasParameter(
     Operation *call, iterator_range<Region::iterator> inlinedBlocks) {
 
-  // First collect all noalias parameters. These have been specially marked by
-  // the `handleArgument` implementation by using the `ssa.copy` intrinsic and
-  // attaching a `noalias` attribute to it.
-  // These are only meant to be temporary and should therefore be deleted after
-  // we're done using them here.
+  // First, collect all ssa copy operations, which correspond to function
+  // parameters, and additionally store the noalias parameters. All parameters
+  // have been marked by the `handleArgument` implementation by using the
+  // `ssa.copy` intrinsic. Additionally, noalias parameters have an attached
+  // `noalias` attribute to the intrinsics. These intrinsics are only meant to
+  // be temporary and should therefore be deleted after we're done using them
+  // here.
+  SetVector<LLVM::SSACopyOp> ssaCopies;
   SetVector<LLVM::SSACopyOp> noAliasParams;
   for (Value argument : cast<LLVM::CallOp>(call).getArgOperands()) {
     for (Operation *user : argument.getUsers()) {
       auto ssaCopy = llvm::dyn_cast<LLVM::SSACopyOp>(user);
       if (!ssaCopy)
         continue;
+      ssaCopies.insert(ssaCopy);
+
       if (!ssaCopy->hasAttr(LLVM::LLVMDialect::getNoAliasAttrName()))
         continue;
-
       noAliasParams.insert(ssaCopy);
     }
   }
 
-  // If there were none, we have nothing to do here.
-  if (noAliasParams.empty())
-    return;
-
   // Scope exit block to make it impossible to forget to get rid of the
   // intrinsics.
   auto exit = llvm::make_scope_exit([&] {
-    for (LLVM::SSACopyOp ssaCopyOp : noAliasParams) {
+    for (LLVM::SSACopyOp ssaCopyOp : ssaCopies) {
       ssaCopyOp.replaceAllUsesWith(ssaCopyOp.getOperand());
       ssaCopyOp->erase();
     }
   });
 
+  // If there were no noalias parameters, we have nothing to do here.
+  if (noAliasParams.empty())
+    return;
+
   // Create a new domain for this specific inlining and a new scope for every
   // noalias parameter.
   auto functionDomain = LLVM::AliasScopeDomainAttr::get(
@@ -335,7 +339,7 @@ static void createNewAliasScopesFromNoAliasParameter(
       bool aliasesOtherKnownObject = false;
       // Go through the based on pointers and check that they are either:
       // * Constants that can be ignored (undef, poison, null pointer).
-      // * Based on a noalias parameter.
+      // * Based on a pointer parameter.
       // * Other pointers that we know can't alias with our noalias parameter.
       //
       // Any other value might be a pointer based on any noalias parameter that
@@ -346,11 +350,13 @@ static void createNewAliasScopesFromNoAliasParameter(
             if (matchPattern(object, m_Constant()))
               return false;
 
-            if (noAliasParams.contains(object.getDefiningOp<LLVM::SSACopyOp>()))
+            if (auto ssaCopy = object.getDefiningOp<LLVM::SSACopyOp>()) {
+              // If that value is based on a noalias parameter, it is guaranteed
+              // to not alias with any other object.
+              aliasesOtherKnownObject |= !noAliasParams.contains(ssaCopy);
               return false;
+            }
 
-            // TODO: This should include other arguments from the inlined
-            //       callable.
             if (isa_and_nonnull<LLVM::AllocaOp, LLVM::AddressOfOp>(
                     object.getDefiningOp())) {
               aliasesOtherKnownObject = true;
@@ -773,29 +779,25 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
       return handleByValArgument(builder, callable, argument, elementType,
                                  requestedAlignment);
     }
-    if (argumentAttrs.contains(LLVM::LLVMDialect::getNoAliasAttrName())) {
-      if (argument.use_empty())
-        return argument;
-
-      // This code is essentially a workaround for deficiencies in the
-      // inliner interface: We need to transform operations *after* inlined
-      // based on the argument attributes of the parameters *before* inlining.
-      // This method runs prior to actual inlining and thus cannot transform the
-      // post-inlining code, while `processInlinedCallBlocks` does not have
-      // access to pre-inlining function arguments. Additionally, it is required
-      // to distinguish which parameter an SSA value originally came from.
-      // As a workaround until this is changed: Create an ssa.copy intrinsic
-      // with the noalias attribute that can easily be found, and is extremely
-      // unlikely to exist in the code prior to inlining, using this to
-      // communicate between this method and `processInlinedCallBlocks`.
-      // TODO: Fix this by refactoring the inliner interface.
-      auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
+
+    // This code is essentially a workaround for deficiencies in the inliner
+    // interface: We need to transform operations *after* inlined based on the
+    // argument attributes of the parameters *before* inlining. This method runs
+    // prior to actual inlining and thus cannot transform the post-inlining
+    // code, while `processInlinedCallBlocks` does not have access to
+    // pre-inlining function arguments. Additionally, it is required to
+    // distinguish which parameter an SSA value originally came from. As a
+    // workaround until this is changed: Create an ssa.copy intrinsic with the
+    // noalias attribute (when it was present before) that can easily be found,
+    // and is extremely unlikely to exist in the code prior to inlining, using
+    // this to communicate between this method and `processInlinedCallBlocks`.
+    // TODO: Fix this by refactoring the inliner interface.
+    auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
+    if (argumentAttrs.contains(LLVM::LLVMDialect::getNoAliasAttrName()))
       copyOp->setDiscardableAttr(
           builder.getStringAttr(LLVM::LLVMDialect::getNoAliasAttrName()),
           builder.getUnitAttr());
-      return copyOp;
-    }
-    return argument;
+    return copyOp;
   }
 
   void processInlinedCallBlocks(
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index a101552e419bc87..775ed8f37344ed3 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -2855,8 +2855,8 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
 LogicalResult WinogradFilterTransformOp::verify() {
   auto filterType = cast<ShapedType>(getFilter().getType());
   ArrayRef<int64_t> filterShape = filterType.getShape();
-  int64_t filterH = filterShape[1];
-  int64_t filterW = filterShape[2];
+  int64_t filterH = filterShape[getFilterHDim()];
+  int64_t filterW = filterShape[getFilterWDim()];
   int64_t r = getR();
   int64_t m = getM();
 
@@ -2870,8 +2870,8 @@ LogicalResult WinogradFilterTransformOp::verify() {
   SmallVector<int64_t> expectedOutputShape;
   expectedOutputShape.push_back(filterH == r ? m + r - 1 : 1);
   expectedOutputShape.push_back(filterW == r ? m + r - 1 : 1);
-  expectedOutputShape.push_back(filterShape[3]);
-  expectedOutputShape.push_back(filterShape[0]);
+  expectedOutputShape.push_back(filterShape[getFilterCDim()]);
+  expectedOutputShape.push_back(filterShape[getFilterFDim()]);
 
   auto outputType = cast<ShapedType>(getOutput().getType());
   ArrayRef<int64_t> outputShape = outputType.getShape();
@@ -2881,6 +2881,103 @@ LogicalResult WinogradFilterTransformOp::verify() {
   return success();
 }
 
+SmallVector<Range>
+WinogradFilterTransformOp::getIterationDomain(OpBuilder &builder) {
+  Location loc = getLoc();
+  IntegerAttr zeroAttr = builder.getIndexAttr(0);
+  IntegerAttr oneAttr = builder.getIndexAttr(1);
+  Value filter = getFilter();
+  int64_t filterRank = getFilterOperandRank();
+  SmallVector<Range> loopBounds(filterRank);
+  for (unsigned dim = 0; dim < filterRank; ++dim) {
+    loopBounds[dim].offset = zeroAttr;
+    loopBounds[dim].size = getDimValue(builder, loc, filter, dim);
+    loopBounds[dim].stride = oneAttr;
+  }
+  return loopBounds;
+}
+
+SmallVector<utils::IteratorType>
+WinogradFilterTransformOp::getLoopIteratorTypes() {
+  int64_t filterRank = getFilterOperandRank();
+  SmallVector<utils::IteratorType> iteratorTypes(filterRank,
+                                                 utils::IteratorType::parallel);
+  return iteratorTypes;
+}
+
+LogicalResult WinogradFilterTransformOp::getResultTilePosition(
+    OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
+    SmallVector<OpFoldResult> &resultSizes) {
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  ShapedType filterType = getFilterOperandType();
+  ArrayRef<int64_t> filterShape = filterType.getShape();
+  int64_t filterH = filterShape[getFilterHDim()];
+  int64_t filterW = filterShape[getFilterWDim()];
+  int64_t m = getM();
+  int64_t r = getR();
+  int64_t alpha = m + r - 1;
+  int64_t alphaH = filterH != 1 ? alpha : 1;
+  int64_t alphaW = filterW != 1 ? alpha : 1;
+  IntegerAttr alphaHAttr = builder.getI64IntegerAttr(alphaH);
+  IntegerAttr alphaWAttr = builder.getI64IntegerAttr(alphaW);
+
+  resultOffsets.append(
+      {zeroAttr, zeroAttr, offsets[getFilterCDim()], offsets[getFilterFDim()]});
+  resultSizes.append(
+      {alphaHAttr, alphaWAttr, sizes[getFilterCDim()], sizes[getFilterFDim()]});
+
+  return success();
+}
+
+/// Implement tiling for winograd_filter_transform
+/// The input of winograd_filter_transform is (F, KH, KW, C).
+/// The output of winograd_filter_transform is (alphaH, alphaW, C, F)
+/// Users can specify the tile sizes of F and C.
+/// `offsets` are the values for the offsets of F, KH, KW, C for one tile.
+/// `sizes` are the values for the sizes of F, KH, KW, C for one tile.
+FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
+    OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes) {
+  IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  ShapedType filterType = getFilterOperandType();
+  ArrayRef<int64_t> filterShape = filterType.getShape();
+  int64_t filterH = filterShape[getFilterHDim()];
+  int64_t filterW = filterShape[getFilterWDim()];
+  IntegerAttr filterHAttr = builder.getI64IntegerAttr(filterH);
+  IntegerAttr filterWAttr = builder.getI64IntegerAttr(filterW);
+  SmallVector<Value> tiledOperands;
+  SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
+
+  sliceOffsets.append(
+      {offsets[getFilterFDim()], zeroAttr, zeroAttr, offsets[getFilterCDim()]});
+  sliceSizes.append({sizes[getFilterFDim()], filterHAttr, filterWAttr,
+                     sizes[getFilterCDim()]});
+  int64_t filterRank = getFilterOperandRank();
+  SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
+  Location loc = getLoc();
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getFilter(), sliceOffsets, sliceSizes, filterStrides));
+
+  SmallVector<OpFoldResult> resultOffsets, resultSizes;
+  if (failed(getResultTilePosition(builder, 1, offsets, sizes, resultOffsets,
+                                   resultSizes)))
+    return failure();
+
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getOutput(), resultOffsets, resultSizes, outputStrides));
+
+  SmallVector<Type> resultTypes;
+  resultTypes.push_back(tiledOperands[1].getType());
+  Operation *tiledOp =
+      mlir::clone(builder, getOperation(), resultTypes, tiledOperands);
+
+  return TilingResult{{tiledOp}, SmallVector<Value>(tiledOp->getResults())};
+}
+
 //===----------------------------------------------------------------------===//
 // WinogradInputTransformOp
 //===----------------------------------------------------------------------===//
@@ -2888,8 +2985,8 @@ LogicalResult WinogradFilterTransformOp::verify() {
 LogicalResult WinogradInputTransformOp::verify() {
   auto inputType = cast<ShapedType>(getInput().getType());
   ArrayRef<int64_t> inputShape = inputType.getShape();
-  int64_t inputH = inputShape[1];
-  int64_t inputW = inputShape[2];
+  int64_t inputH = inputShape[getInputHDim()];
+  int64_t inputW = inputShape[getInputWDim()];
   int m = getM();
   int r = getR();
   int64_t tileSize = m + r - 1;
@@ -2898,21 +2995,23 @@ LogicalResult WinogradInputTransformOp::verify() {
 
   SmallVector<int64_t> expectedOutputShape(6, inputH);
   if (ShapedType::isDynamic(inputH)) {
-    expectedOutputShape[0] = tileSize;
-    expectedOutputShape[2] = ShapedType::kDynamic;
+    expectedOutputShape[getOutputAlphaHDim()] = tileSize;
+    expectedOutputShape[getOutputTileHDim()] = ShapedType::kDynamic;
   } else {
-    expectedOutputShape[0] = leftTransform ? tileSize : 1;
-    expectedOutputShape[2] = leftTransform ? (inputH - (r - 1)) / m : 1;
+    expectedOutputShape[getOutputAlphaHDim()] = leftTransform ? tileSize : 1;
+    expectedOutputShape[getOutputTileHDim()] =
+        leftTransform ? (inputH - (r - 1)) / m : 1;
   }
   if (ShapedType::isDynamic(inputW)) {
-    expectedOutputShape[1] = tileSize;
-    expectedOutputShape[3] = ShapedType::kDynamic;
+    expectedOutputShape[getOutputAlphaWDim()] = tileSize;
+    expectedOutputShape[getOutputTileWDim()] = ShapedType::kDynamic;
   } else {
-    expectedOutputShape[1] = rightTransform ? tileSize : 1;
-    expectedOutputShape[3] = rightTransform ? (inputW - (r - 1)) / m : 1;
+    expectedOutputShape[getOutputAlphaWDim()] = rightTransform ? tileSize : 1;
+    expectedOutputShape[getOutputTileWDim()] =
+        rightTransform ? (inputW - (r - 1)) / m : 1;
   }
-  expectedOutputShape[4] = inputShape[0];
-  expectedOutputShape[5] = inputShape[3];
+  expectedOutputShape[getOutputNDim()] = inputShape[getInputNDim()];
+  expectedOutputShape[getOutputCDim()] = inputShape[getInputCDim()];
 
   auto outputType = cast<ShapedType>(getOutput().getType());
   ArrayRef<int64_t> outputShape = outputType.getShape();
@@ -2922,6 +3021,130 @@ LogicalResult WinogradInputTransformOp::verify() {
   return success();
 }
 
+SmallVector<Range>
+WinogradInputTransformOp::getIterationDomain(OpBuilder &builder) {
+  Location loc = getLoc();
+  IntegerAttr zeroAttr = builder.getIndexAttr(0);
+  IntegerAttr oneAttr = builder.getIndexAttr(1);
+  Value output = getOutput();
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<Range> loopBounds(outputRank);
+  for (unsigned dim = 0; dim < outputRank; ++dim) {
+    loopBounds[dim].offset = zeroAttr;
+    // alphaH, alphaW, tileH, tileW, N, C
+    loopBounds[dim].size = getDimValue(builder, loc, output, dim);
+    loopBounds[dim].stride = oneAttr;
+  }
+  return loopBounds;
+}
+
+SmallVector<utils::IteratorType>
+WinogradInputTransformOp::getLoopIteratorTypes() {
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<utils::IteratorType> iteratorTypes(outputRank,
+                                                 utils::IteratorType::parallel);
+  return iteratorTypes;
+}
+
+LogicalResult WinogradInputTransformOp::getResultTilePosition(
+    OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
+    SmallVector<OpFoldResult> &resultSizes) {
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  ShapedType inputType = getInputOperandType();
+  ArrayRef<int64_t> inputShape = inputType.getShape();
+  int64_t inputH = inputShape[getInputHDim()];
+  int64_t inputW = inputShape[getInputWDim()];
+  int64_t m = getM();
+  int64_t r = getR();
+  int64_t alpha = m + r - 1;
+  int64_t alphaH = inputH != 1 ? alpha : 1;
+  int64_t alphaW = inputW != 1 ? alpha : 1;
+  IntegerAttr alphaHAttr = builder.getI64IntegerAttr(alphaH);
+  IntegerAttr alphaWAttr = builder.getI64IntegerAttr(alphaW);
+
+  resultOffsets.append({zeroAttr, zeroAttr, offsets[getOutputTileHDim()],
+                        offsets[getOutputTileWDim()], offsets[getOutputNDim()],
+                        offsets[getOutputCDim()]});
+  resultSizes.append({alphaHAttr, alphaWAttr, sizes[getOutputTileHDim()],
+                      sizes[getOutputTileWDim()], sizes[getOutputNDim()],
+                      sizes[getOutputCDim()]});
+
+  return success();
+}
+
+/// Implement tiling for winograd_input_transform
+/// The input of winograd_input_transform is (N, H, W, C).
+/// The output of winograd_input_transform is (alphaH, alphaW, tileH, tileW, N,
+/// C) Users can specify the tile sizes of tileH, tileW, N, and C. `offsets` are
+/// the values for the offsets of tileH, tileW, N, C for one tile. `sizes` are
+/// the values for the sizes of tileH, tileW, N, C for one tile.
+FailureOr<TilingResult>
+WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
+                                                 ArrayRef<OpFoldResult> offsets,
+                                                 ArrayRef<OpFoldResult> sizes) {
+  IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  ShapedType inputType = getInputOperandType();
+  ArrayRef<int64_t> inputShape = inputType.getShape();
+  int64_t inputH = inputShape[getInputHDim()];
+  int64_t inputW = inputShape[getInputWDim()];
+  int64_t m = getM();
+  int64_t r = getR();
+
+  Location loc = getLoc();
+  MLIRContext *context = builder.getContext();
+  auto offsetAffineMap =
+      AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
+  Value mappedOffsetH = affine::makeComposedAffineApply(
+      builder, loc, offsetAffineMap, offsets[getOutputTileHDim()]);
+  Value mappedOffsetW = affine::makeComposedAffineApply(
+      builder, loc, offsetAffineMap, offsets[getOutputTileWDim()]);
+  auto sizeAffineMap = AffineMap::get(
+      1, 0, {builder.getAffineDimExpr(0) * m + (r - 1)}, context);
+  Value mappedSizeH = affine::makeComposedAffineApply(
+      builder, loc, sizeAffineMap, sizes[getOutputTileHDim()]);
+  Value mappedSizeW = affine::makeComposedAffineApply(
+      builder, loc, sizeAffineMap, sizes[getOutputTileWDim()]);
+
+  SmallVector<Value> tiledOperands;
+  SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
+
+  OpFoldResult offsetH =
+      inputH != 1 ? OpFoldResult(mappedOffsetH) : OpFoldResult(zeroAttr);
+  OpFoldResult offsetW =
+      inputW != 1 ? OpFoldResult(mappedOffsetW) : OpFoldResult(zeroAttr);
+  sliceOffsets.append(
+      {offsets[getOutputNDim()], offsetH, offsetW, offsets[getOutputCDim()]});
+  OpFoldResult sizeH =
+      inputH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
+  OpFoldResult sizeW =
+      inputW != 1 ? OpFoldResult(mappedSizeW) : OpFoldResult(oneAttr);
+  sliceSizes.append(
+      {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
+  int64_t inputRank = getInputOperandRank();
+  SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getInput(), sliceOffsets, sliceSizes, inputStrides));
+
+  SmallVector<OpFoldResult> resultOffsets, resultSizes;
+  if (failed(getResultTilePosition(builder, 1, offsets, sizes, resultOffsets,
+                                   resultSizes)))
+    return failure();
+
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getOutput(), resultOffsets, resultSizes, outputStrides));
+
+  SmallVector<Type> resultTypes;
+  resultTypes.push_back(tiledOperands[1].getType());
+  Operation *tiledOp =
+      mlir::clone(builder, getOperation(), resultTypes, tiledOperands);
+
+  return TilingResult{{tiledOp}, SmallVector<Value>(tiledOp->getResults())};
+}
+
 //===----------------------------------------------------------------------===//
 // WinogradOutputTransformOp
 //===----------------------------------------------------------------------===//
@@ -2929,32 +3152,34 @@ LogicalResult WinogradInputTransformOp::verify() {
 LogicalResult WinogradOutputTransformOp::verify() {
   auto valueType = cast<ShapedType>(getValue().getType());
   ArrayRef<int64_t> valueShape = valueType.getShape();
-  int64_t valueH = valueShape[0];
-  int64_t valueW = valueShape[1];
-  int64_t valueTileH = valueShape[2];
-  int64_t valueTileW = valueShape[3];
+  int64_t valueH = valueShape[getValueAlphaHDim()];
+  int64_t valueW = valueShape[getValueAlphaWDim()];
+  int64_t valueTileH = valueShape[getValueTileHDim()];
+  int64_t valueTileW = valueShape[getValueTileWDim()];
   int m = getM();
   int r = getR();
   bool leftTransform = valueH != 1;
   bool rightTransform = valueW != 1;
 
-  SmallVector<int64_t> expectedOutputShape(4, valueH);
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<int64_t> expectedOutputShape(outputRank, valueH);
   if (ShapedType::isDynamic(valueH) || ShapedType::isDynamic(valueTileH)) {
-    expectedOutputShape[1] = ShapedType::kDynamic;
+    expectedOutputShape[getOutputHDim()] = ShapedType::kDynamic;
   } else {
     if (valueH != (leftTransform ? m + r - 1 : 1))
       return emitOpError("expect input height equals to input tile size");
-    expectedOutputShape[1] = (leftTransform ? m : 1) * valueTileH;
+    expectedOutputShape[getOutputHDim()] = (leftTransform ? m : 1) * valueTileH;
   }
   if (ShapedType::isDynamic(valueW) || ShapedType::isDynamic(valueTileW)) {
-    expectedOutputShape[2] = ShapedType::kDynamic;
+    expectedOutputShape[getOutputWDim()] = ShapedType::kDynamic;
   } else {
     if (valueW != (rightTransform ? m + r - 1 : 1))
       return emitOpError("expect input width equals to input tile size");
-    expectedOutputShape[2] = (rightTransform ? m : 1) * valueTileW;
+    expectedOutputShape[getOutputWDim()] =
+        (rightTransform ? m : 1) * valueTileW;
   }
-  expectedOutputShape[0] = valueShape[4];
-  expectedOutputShape[3] = valueShape[5];
+  expectedOutputShape[getOutputNDim()] = valueShape[getValueNDim()];
+  expectedOutputShape[getOutputFDim()] = valueShape[getValueFDim()];
 
   auto outputType = cast<ShapedType>(getOutput().getType());
   ArrayRef<int64_t> outputShape = outputType.getShape();
@@ -2964,6 +3189,124 @@ LogicalResult WinogradOutputTransformOp::verify() {
   return success();
 }
 
+SmallVector<Range>
+WinogradOutputTransformOp::getIterationDomain(OpBuilder &builder) {
+  Location loc = getLoc();
+  IntegerAttr zeroAttr = builder.getIndexAttr(0);
+  IntegerAttr oneAttr = builder.getIndexAttr(1);
+  Value value = getValue();
+  int64_t valueRank = getValueOperandRank();
+  SmallVector<Range> loopBounds(valueRank);
+  for (unsigned dim = 0; dim < valueRank; ++dim) {
+    loopBounds[dim].offset = zeroAttr;
+    // alphaH, alphaW, tileH, tileW, N, F
+    loopBounds[dim].size = getDimValue(builder, loc, value, dim);
+    loopBounds[dim].stride = oneAttr;
+  }
+  return loopBounds;
+}
+
+SmallVector<utils::IteratorType>
+WinogradOutputTransformOp::getLoopIteratorTypes() {
+  int64_t valueRank = getValueOperandRank();
+  SmallVector<utils::IteratorType> iteratorTypes(valueRank,
+                                                 utils::IteratorType::parallel);
+  return iteratorTypes;
+}
+
+LogicalResult WinogradOutputTransformOp::getResultTilePosition(
+    OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
+    SmallVector<OpFoldResult> &resultSizes) {
+  int64_t m = getM();
+
+  Location loc = getLoc();
+  MLIRContext *context = builder.getContext();
+  auto affineMap =
+      AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
+
+  Value mappedOffsetH = affine::makeComposedAffineApply(
+      builder, loc, affineMap, offsets[getValueTileHDim()]);
+  Value mappedOffsetW = affine::makeComposedAffineApply(
+      builder, loc, affineMap, offsets[getValueTileWDim()]);
+  Value mappedSizeH = affine::makeComposedAffineApply(
+      builder, loc, affineMap, sizes[getValueTileHDim()]);
+  Value mappedSizeW = affine::makeComposedAffineApply(
+      builder, loc, affineMap, sizes[getValueTileWDim()]);
+
+  ShapedType valueType = getValueOperandType();
+  ArrayRef<int64_t> valueShape = valueType.getShape();
+  int64_t valueH = valueShape[0];
+  int64_t valueW = valueShape[1];
+  IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  OpFoldResult offsetH =
+      valueH != 1 ? OpFoldResult(mappedOffsetH) : OpFoldResult(zeroAttr);
+  OpFoldResult offsetW =
+      valueW != 1 ? OpFoldResult(mappedOffsetW) : OpFoldResult(zeroAttr);
+  OpFoldResult sizeH =
+      valueH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
+  OpFoldResult sizeW =
+      valueW != 1 ? OpFoldResult(mappedSizeW) : OpFoldResult(oneAttr);
+
+  resultOffsets.append(
+      {offsets[getValueNDim()], offsetH, offsetW, offsets[getValueFDim()]});
+  resultSizes.append(
+      {sizes[getValueNDim()], sizeH, sizeW, sizes[getValueFDim()]});
+  return success();
+}
+
+/// Implement tiling for winograd_output_transform
+/// The input of winograd_output_transform is (alphaH, alphaW, tileH, tileW, N,
+/// F). The output of winograd_output_transform is (N, H, W, F) Users can
+/// specify the tile sizes of tileH, tileW, N, and F. `offsets` are the values
+/// for the offsets of tileH, tileW, N, F for one tile. `sizes` are the values
+/// for the sizes of tileH, tileW, N, F for one tile.
+FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
+    OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes) {
+  IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  Location loc = getLoc();
+  SmallVector<Value> tiledOperands;
+  SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
+
+  ShapedType valueType = getValueOperandType();
+  ArrayRef<int64_t> valueShape = valueType.getShape();
+  int64_t alphaH = valueShape[getValueAlphaHDim()];
+  int64_t alphaW = valueShape[getValueAlphaWDim()];
+  IntegerAttr alphaHAttr = builder.getI64IntegerAttr(alphaH);
+  IntegerAttr alphaWAttr = builder.getI64IntegerAttr(alphaW);
+
+  sliceOffsets.append({zeroAttr, zeroAttr, offsets[getValueTileHDim()],
+                       offsets[getValueTileWDim()], offsets[getValueNDim()],
+                       offsets[getValueFDim()]});
+  sliceSizes.append({alphaHAttr, alphaWAttr, sizes[getValueTileHDim()],
+                     sizes[getValueTileWDim()], sizes[getValueNDim()],
+                     sizes[getValueFDim()]});
+  int64_t valueRank = getValueOperandRank();
+  SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getValue(), sliceOffsets, sliceSizes, sliceStrides));
+
+  SmallVector<OpFoldResult> resultOffsets, resultSizes;
+  if (failed(getResultTilePosition(builder, 1, offsets, sizes, resultOffsets,
+                                   resultSizes)))
+    return failure();
+
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<OpFoldResult> strides(outputRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getOutput(), resultOffsets, resultSizes, strides));
+
+  SmallVector<Type> resultTypes;
+  resultTypes.push_back(tiledOperands[1].getType());
+  Operation *tiledOp =
+      mlir::clone(builder, getOperation(), resultTypes, tiledOperands);
+
+  return TilingResult{{tiledOp}, SmallVector<Value>(tiledOp->getResults())};
+}
+
 //===----------------------------------------------------------------------===//
 // LinalgDialect
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 48b3abbeee7010c..29b5631f61b4885 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -3452,7 +3452,7 @@ transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
   if (!getDisableMultiReductionToContractPatterns())
     vector::populateVectorReductionToContractPatterns(patterns);
 
-  vector::populateSinkVectorBroadcastPatterns(patterns);
+  vector::populateSinkVectorOpsPatterns(patterns);
 
   patterns.add<linalg::LinalgCopyVTRForwardingPattern,
                linalg::LinalgCopyVTWForwardingPattern>(ctx,
@@ -3851,6 +3851,47 @@ DiagnosedSilenceableFailure transform::WinogradConv2DOp::applyToOne(
   return DiagnosedSilenceableFailure::success();
 }
 
+DiagnosedSilenceableFailure transform::DecomposeWinogradOp::applyToOne(
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
+    transform::TransformState &state) {
+  rewriter.setInsertionPoint(target);
+  FailureOr<Operation *> maybeTransformed = failure();
+  bool supported =
+      TypeSwitch<Operation *, bool>(target)
+          .Case([&](linalg::WinogradFilterTransformOp op) {
+            maybeTransformed = decomposeWinogradFilterTransformOp(rewriter, op);
+            return true;
+          })
+          .Case([&](linalg::WinogradInputTransformOp op) {
+            maybeTransformed = decomposeWinogradInputTransformOp(rewriter, op);
+            return true;
+          })
+          .Case([&](linalg::WinogradOutputTransformOp op) {
+            maybeTransformed = decomposeWinogradOutputTransformOp(rewriter, op);
+            return true;
+          })
+          .Default([&](Operation *op) { return false; });
+
+  if (!supported) {
+    DiagnosedSilenceableFailure diag =
+        emitSilenceableError()
+        << "this operation is not supported to decompose into other operations";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+
+  if (supported && failed(maybeTransformed)) {
+    DiagnosedSilenceableFailure diag =
+        emitSilenceableError() << "decompose Winograd operations failed";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+
+  results.push_back(*maybeTransformed);
+  return DiagnosedSilenceableFailure::success();
+}
+
 #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOpsEnums.cpp.inc"
 
 #define GET_OP_CLASSES
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index c6c770e2781ff07..b65b18699a15aa3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -490,8 +490,6 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
   Type elementType = inputType.getElementType();
   auto inputShape = inputType.getShape(); // N, H, W, C
   int64_t inputN = inputShape[0];
-  int64_t inputH = inputShape[1];
-  int64_t inputW = inputShape[2];
   int64_t inputC = inputShape[3];
   auto valueType = cast<ShapedType>(retValue.getType());
   auto valueShape = valueType.getShape(); // alphaH, alphaW, HTile, WTile, N, C
@@ -500,11 +498,6 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
   int64_t alphaH = leftTransform ? m + r - 1 : 1;
   int64_t alphaW = rightTransform ? m + r - 1 : 1;
 
-  if ((inputH != (tileH * m) + (r - 1)) && inputH != 1)
-    return Value();
-  if ((inputW != (tileW * m) + (r - 1)) && inputW != 1)
-    return Value();
-
   auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
                        ValueRange args) -> scf::ValueVector {
     Value tileHIter = ivs[0];
@@ -1169,6 +1162,24 @@ FailureOr<Operation *> winogradConv2D(RewriterBase &rewriter,
   return winogradConv2DHelper(rewriter, op, m, r);
 }
 
+FailureOr<Operation *>
+decomposeWinogradFilterTransformOp(RewriterBase &rewriter,
+                                   linalg::WinogradFilterTransformOp op) {
+  return decomposeWinogradFilterTransformHelper(rewriter, op);
+}
+
+FailureOr<Operation *>
+decomposeWinogradInputTransformOp(RewriterBase &rewriter,
+                                  linalg::WinogradInputTransformOp op) {
+  return decomposeWinogradInputTransformHelper(rewriter, op);
+}
+
+FailureOr<Operation *>
+decomposeWinogradOutputTransformOp(RewriterBase &rewriter,
+                                   linalg::WinogradOutputTransformOp op) {
+  return decomposeWinogradOutputTransformHelper(rewriter, op);
+}
+
 void populateWinogradConv2DPatterns(RewritePatternSet &patterns, int64_t m,
                                     int64_t r) {
   MLIRContext *context = patterns.getContext();
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 4de8dacc0edbf33..e92d9503372cdf2 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -4322,7 +4322,7 @@ struct FoldConstantCase : OpRewritePattern<scf::IndexSwitchOp> {
 
     rewriter.inlineBlockBefore(&source, op);
     rewriter.eraseOp(terminator);
-    // Repalce the operation with a potentially empty list of results.
+    // Replace the operation with a potentially empty list of results.
     // Fold mechanism doesn't support the case where the result list is empty.
     rewriter.replaceOp(op, results);
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp b/mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp
index fcbef2be75f9a09..2e5a2aab52a1606 100644
--- a/mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp
@@ -150,6 +150,30 @@ LogicalResult GroupNonUniformBallotOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotFindLSBOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult GroupNonUniformBallotFindLSBOp::verify() {
+  spirv::Scope scope = getExecutionScope();
+  if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
+    return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotFindLSBOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult GroupNonUniformBallotFindMSBOp::verify() {
+  spirv::Scope scope = getExecutionScope();
+  if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
+    return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // spirv.GroupNonUniformBroadcast
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 13c26ca4900d42f..5f51d4b25ccc257 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -1088,11 +1088,16 @@ llvm::LogicalResult tosa::ReshapeOp::verify() {
     return emitOpError() << "new shape does not match result rank";
 
   for (auto [newShapeDim, outputShapeDim] :
-       zip(getNewShape(), outputType.getShape()))
+       zip(getNewShape(), outputType.getShape())) {
     if (newShapeDim != -1 && outputShapeDim != ShapedType::kDynamic &&
         newShapeDim != outputShapeDim)
       return emitOpError() << "new shape is inconsistent with result shape";
 
+    if (newShapeDim != ShapedType::kDynamic && newShapeDim < -1)
+      return emitOpError() << "new shape has invalid tensor dimension size "
+                           << newShapeDim;
+  }
+
   if (inputType.hasStaticShape() && outputType.hasStaticShape()) {
     int64_t inputElementsNum = inputType.getNumElements();
     int64_t outputElementsNum = outputType.getNumElements();
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index b1d5720541846f3..358353eb955c16c 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -88,18 +88,36 @@ class TypeModificationState {
     // For each use whose type changed, cast the value with the new type back to
     // the old type.
     for (auto [value, oldType] : oldTypes) {
-      tensor::CastOp castedValue;
-      for (auto &use : value.getUses()) {
-        if (canBeRefined(use.getOwner()))
+      // The call to 'use->set()' in the body of the loop below invalidates the
+      // iterator used to traverse op uses, so it is important to make a copy of
+      // these first.
+      llvm::SmallVector<OpOperand *> uses = llvm::map_to_vector(
+          value.getUses(),
+          [](OpOperand &use) -> OpOperand * {
+            return &use;
+          });
+
+      // A 'tensor.cast' op is emitted only if needed. Once emitted, it is
+      // cached and reused by all consumers.
+      tensor::CastOp castValue;
+
+      // Traverse all uses
+      for (OpOperand *use : uses) {
+        if (canBeRefined(use->getOwner()))
           continue;
 
-        // Cache the cast to avoid generating duplicates
-        if (!castedValue) {
-          ImplicitLocOpBuilder builder{value.getLoc(), use.getOwner()};
-          castedValue = builder.create<tensor::CastOp>(oldType, value);
+        if (!castValue) {
+          // Set the insertion point as far back as possible, since new
+          // consumers of the 'tensor.cast' op generated in future iterations
+          // are likely to be further up in the code due to the order in which
+          // they appear in the use list.
+          OpBuilder builder{value.getContext()};
+          builder.setInsertionPointAfter(value.getDefiningOp());
+          castValue =
+              builder.create<tensor::CastOp>(value.getLoc(), oldType, value);
         }
 
-        use.set(castedValue);
+        use->set(castValue);
       }
     }
 
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index bc423a3781bf045..3ae70ace3934cdd 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -67,6 +67,7 @@ void transform::ApplyFoldElementwiseToVectorPatternsOp::populatePatterns(
 void transform::ApplyVectorReductionToContractPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
   vector::populateVectorReductionToContractPatterns(patterns);
+  vector::populateSinkVectorOpsPatterns(patterns);
 }
 
 void transform::ApplyLowerCreateMaskPatternsOp::populatePatterns(
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index ccbaa3e9759975c..ad4e42b31962e17 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -2030,8 +2030,7 @@ void mlir::vector::populateVectorContractCanonicalizeMatmulToMMT(
 void mlir::vector::populateVectorReductionToContractPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
   patterns.add<MultiReduceToContract, CombineContractBroadcast,
-               CombineContractABTranspose, CombineContractResultTranspose,
-               ReorderCastOpsOnBroadcast, ReorderElementwiseOpsOnTranspose>(
+               CombineContractABTranspose, CombineContractResultTranspose>(
       patterns.getContext(), benefit);
 }
 
@@ -2043,10 +2042,11 @@ void mlir::vector::
                                                    benefit);
 }
 
-void mlir::vector::populateSinkVectorBroadcastPatterns(
-    RewritePatternSet &patterns, PatternBenefit benefit) {
-  patterns.add<ReorderCastOpsOnBroadcast, ReorderElementwiseOpsOnBroadcast>(
-      patterns.getContext(), benefit);
+void mlir::vector::populateSinkVectorOpsPatterns(RewritePatternSet &patterns,
+                                                 PatternBenefit benefit) {
+  patterns.add<ReorderElementwiseOpsOnTranspose, ReorderCastOpsOnBroadcast,
+               ReorderElementwiseOpsOnBroadcast>(patterns.getContext(),
+                                                 benefit);
 }
 
 void mlir::vector::populateChainedVectorReductionFoldingPatterns(
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index b51357198b1ca00..8f28d61c732cdcd 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -619,8 +619,8 @@ static void checkFoldResultTypes(Operation *op,
     if (auto value = dyn_cast<Value>(ofr)) {
       if (value.getType() != opResult.getType()) {
         op->emitOpError() << "folder produced a value of incorrect type: "
-                          << opResult.getType()
-                          << ", expected: " << value.getType();
+                          << value.getType()
+                          << ", expected: " << opResult.getType();
         assert(false && "incorrect fold result type");
       }
     }
diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
index 1dadb9dd691e70a..8e112e6f7dda652 100644
--- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp
+++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
@@ -449,6 +449,43 @@ static LogicalResult printOperation(CppEmitter &emitter, emitc::SubOp subOp) {
   return printBinaryOperation(emitter, operation, "-");
 }
 
+static LogicalResult emitSwitchCase(CppEmitter &emitter,
+                                    raw_indented_ostream &os, Region &region) {
+  for (Region::OpIterator iteratorOp = region.op_begin(), end = region.op_end();
+       std::next(iteratorOp) != end; ++iteratorOp) {
+    if (failed(emitter.emitOperation(*iteratorOp, /*trailingSemicolon=*/true)))
+      return failure();
+  }
+  os << "break;\n";
+  return success();
+}
+
+static LogicalResult printOperation(CppEmitter &emitter,
+                                    emitc::SwitchOp switchOp) {
+  raw_indented_ostream &os = emitter.ostream();
+
+  os << "\nswitch (" << emitter.getOrCreateName(switchOp.getArg()) << ") {";
+
+  for (auto pair : llvm::zip(switchOp.getCases(), switchOp.getCaseRegions())) {
+    os << "\ncase " << std::get<0>(pair) << ": {\n";
+    os.indent();
+
+    if (failed(emitSwitchCase(emitter, os, std::get<1>(pair))))
+      return failure();
+
+    os.unindent() << "}";
+  }
+
+  os << "\ndefault: {\n";
+  os.indent();
+
+  if (failed(emitSwitchCase(emitter, os, switchOp.getDefaultRegion())))
+    return failure();
+
+  os.unindent() << "}";
+  return success();
+}
+
 static LogicalResult printOperation(CppEmitter &emitter, emitc::CmpOp cmpOp) {
   Operation *operation = cmpOp.getOperation();
 
@@ -998,7 +1035,7 @@ static LogicalResult printFunctionBody(CppEmitter &emitter,
       // trailing semicolon is handled within the printOperation function.
       bool trailingSemicolon =
           !isa<cf::CondBranchOp, emitc::DeclareFuncOp, emitc::ForOp,
-               emitc::IfOp, emitc::VerbatimOp>(op);
+               emitc::IfOp, emitc::SwitchOp, emitc::VerbatimOp>(op);
 
       if (failed(emitter.emitOperation(
               op, /*trailingSemicolon=*/trailingSemicolon)))
@@ -1509,8 +1546,8 @@ LogicalResult CppEmitter::emitOperation(Operation &op, bool trailingSemicolon) {
                 emitc::GlobalOp, emitc::IfOp, emitc::IncludeOp,
                 emitc::LogicalAndOp, emitc::LogicalNotOp, emitc::LogicalOrOp,
                 emitc::MulOp, emitc::RemOp, emitc::ReturnOp, emitc::SubOp,
-                emitc::UnaryMinusOp, emitc::UnaryPlusOp, emitc::VariableOp,
-                emitc::VerbatimOp>(
+                emitc::SwitchOp, emitc::UnaryMinusOp, emitc::UnaryPlusOp,
+                emitc::VariableOp, emitc::VerbatimOp>(
               [&](auto op) { return printOperation(*this, op); })
           // Func ops.
           .Case<func::CallOp, func::FuncOp, func::ReturnOp>(
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 458d05d5059db70..1f3fb95c339c7c2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -690,14 +690,14 @@ inlineOmpRegionCleanup(llvm::SmallVectorImpl<Region *> &cleanupRegions,
                                           : &builder.GetInsertBlock()->back();
     if (potentialTerminator && potentialTerminator->isTerminator())
       builder.SetInsertPoint(potentialTerminator);
-    llvm::Value *prviateVarValue =
+    llvm::Value *privateVarValue =
         shouldLoadCleanupRegionArg
             ? builder.CreateLoad(
                   moduleTranslation.convertType(entry.getArgument(0).getType()),
                   privateVariables[i])
             : privateVariables[i];
 
-    moduleTranslation.mapValue(entry.getArgument(0), prviateVarValue);
+    moduleTranslation.mapValue(entry.getArgument(0), privateVarValue);
 
     if (failed(inlineConvertOmpRegions(*cleanupRegion, regionName, builder,
                                        moduleTranslation)))
@@ -1424,35 +1424,106 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
     }
   };
 
-  SmallVector<omp::PrivateClauseOp> privatizerClones;
-  SmallVector<llvm::Value *> privateVariables;
+  SmallVector<omp::PrivateClauseOp> mlirPrivatizerClones;
+  SmallVector<llvm::Value *> llvmPrivateVars;
 
   // TODO: Perform appropriate actions according to the data-sharing
   // attribute (shared, private, firstprivate, ...) of variables.
   // Currently shared and private are supported.
   auto privCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
-                    llvm::Value &, llvm::Value &vPtr,
-                    llvm::Value *&replacementValue) -> InsertPointTy {
-    replacementValue = &vPtr;
+                    llvm::Value &, llvm::Value &llvmOmpRegionInput,
+                    llvm::Value *&llvmReplacementValue) -> InsertPointTy {
+    llvmReplacementValue = &llvmOmpRegionInput;
 
     // If this is a private value, this lambda will return the corresponding
     // mlir value and its `PrivateClauseOp`. Otherwise, empty values are
     // returned.
-    auto [privVar, privatizerClone] =
+    auto [mlirPrivVar, mlirPrivatizerClone] =
         [&]() -> std::pair<mlir::Value, omp::PrivateClauseOp> {
       if (!opInst.getPrivateVars().empty()) {
-        auto privateVars = opInst.getPrivateVars();
-        auto privateSyms = opInst.getPrivateSyms();
+        auto mlirPrivVars = opInst.getPrivateVars();
+        auto mlirPrivSyms = opInst.getPrivateSyms();
 
-        for (auto [privVar, privatizerAttr] :
-             llvm::zip_equal(privateVars, *privateSyms)) {
+        // Try to find a privatizer that corresponds to the LLVM value being
+        // privatized.
+        for (auto [mlirPrivVar, mlirPrivatizerAttr] :
+             llvm::zip_equal(mlirPrivVars, *mlirPrivSyms)) {
           // Find the MLIR private variable corresponding to the LLVM value
           // being privatized.
-          llvm::Value *llvmPrivVar = moduleTranslation.lookupValue(privVar);
-          if (llvmPrivVar != &vPtr)
+          llvm::Value *mlirToLLVMPrivVar =
+              moduleTranslation.lookupValue(mlirPrivVar);
+
+          // Check if the LLVM value being privatized matches the LLVM value
+          // mapped to privVar. In some cases, this is not trivial ...
+          auto isMatch = [&]() {
+            if (mlirToLLVMPrivVar == nullptr)
+              return false;
+
+            // If both values are trivially equal, we found a match.
+            if (mlirToLLVMPrivVar == &llvmOmpRegionInput)
+              return true;
+
+            // Otherwise, we check if both llvmOmpRegionInputPtr and
+            // mlirToLLVMPrivVar refer to the same memory (through a load/store
+            // pair). This happens if a struct (i.e. multi-field value) is being
+            // privatized.
+            //
+            // For example, if the privatized value is defined by:
+            // ```
+            //   %priv_val = alloca { ptr, i64 }, align 8
+            // ```
+            //
+            // The initialization of this value (outside the omp region) will be
+            // something like this:
+            //
+            // clang-format off
+            // ```
+            //   %partially_init_priv_val = insertvalue { ptr, i64 } undef,
+            //                              ptr %some_ptr, 0
+            //   %fully_init_priv_val = insertvalue { ptr, i64 } %partially_init_priv_val,
+            //                          i64 %some_i64, 1
+            //   ...
+            //   store { ptr, i64 } %fully_init_priv_val, ptr %priv_val, align 8
+            // ```
+            // clang-format on
+            //
+            // Now, we enter the OMP region, in order to access this privatized
+            // value, we need to load from the allocated memory:
+            // ```
+            // omp.par.entry:
+            //   %priv_val_load = load { ptr, i64 }, ptr %priv_val, align 8
+            // ```
+            //
+            // The 2 LLVM values tracked here map as follows:
+            // - `mlirToLLVMPrivVar`     -> `%fully_init_priv_val`
+            // - `llvmOmpRegionInputPtr` -> `%priv_val_load`
+            //
+            // Even though they eventually refer to the same memory reference
+            // (the memory being privatized), they are 2 distinct LLVM values.
+            // Therefore, we need to discover their correspondence by finding
+            // out if they store into and load from the same mem ref.
+            auto *llvmOmpRegionInputPtrLoad =
+                llvm::dyn_cast_if_present<llvm::LoadInst>(&llvmOmpRegionInput);
+
+            if (llvmOmpRegionInputPtrLoad == nullptr)
+              return false;
+
+            for (auto &use : mlirToLLVMPrivVar->uses()) {
+              auto *mlirToLLVMPrivVarStore =
+                  llvm::dyn_cast_if_present<llvm::StoreInst>(use.getUser());
+              if (mlirToLLVMPrivVarStore &&
+                  (llvmOmpRegionInputPtrLoad->getPointerOperand() ==
+                   mlirToLLVMPrivVarStore->getPointerOperand()))
+                return true;
+            }
+
+            return false;
+          };
+
+          if (!isMatch())
             continue;
 
-          SymbolRefAttr privSym = llvm::cast<SymbolRefAttr>(privatizerAttr);
+          SymbolRefAttr privSym = llvm::cast<SymbolRefAttr>(mlirPrivatizerAttr);
           omp::PrivateClauseOp privatizer =
               SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
                   opInst, privSym);
@@ -1480,24 +1551,24 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
               counter);
 
           clone.setSymName(cloneName);
-          return {privVar, clone};
+          return {mlirPrivVar, clone};
         }
       }
 
       return {mlir::Value(), omp::PrivateClauseOp()};
     }();
 
-    if (privVar) {
-      Region &allocRegion = privatizerClone.getAllocRegion();
+    if (mlirPrivVar) {
+      Region &allocRegion = mlirPrivatizerClone.getAllocRegion();
 
       // If this is a `firstprivate` clause, prepare the `omp.private` op by:
-      if (privatizerClone.getDataSharingType() ==
+      if (mlirPrivatizerClone.getDataSharingType() ==
           omp::DataSharingClauseType::FirstPrivate) {
         auto oldAllocBackBlock = std::prev(allocRegion.end());
         omp::YieldOp oldAllocYieldOp =
             llvm::cast<omp::YieldOp>(oldAllocBackBlock->getTerminator());
 
-        Region &copyRegion = privatizerClone.getCopyRegion();
+        Region &copyRegion = mlirPrivatizerClone.getCopyRegion();
 
         mlir::IRRewriter copyCloneBuilder(&moduleTranslation.getContext());
         // 1. Cloning the `copy` region to the end of the `alloc` region.
@@ -1524,7 +1595,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       // This way, the body of the privatizer will be changed from using the
       // region/block argument to the value being privatized.
       auto allocRegionArg = allocRegion.getArgument(0);
-      replaceAllUsesInRegionWith(allocRegionArg, privVar, allocRegion);
+      replaceAllUsesInRegionWith(allocRegionArg, mlirPrivVar, allocRegion);
 
       auto oldIP = builder.saveIP();
       builder.restoreIP(allocaIP);
@@ -1535,15 +1606,15 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
         opInst.emitError("failed to inline `alloc` region of an `omp.private` "
                          "op in the parallel region");
         bodyGenStatus = failure();
-        privatizerClone.erase();
+        mlirPrivatizerClone.erase();
       } else {
         assert(yieldedValues.size() == 1);
-        replacementValue = yieldedValues.front();
+        llvmReplacementValue = yieldedValues.front();
 
         // Keep the LLVM replacement value and the op clone in case we need to
         // emit cleanup (i.e. deallocation) logic.
-        privateVariables.push_back(replacementValue);
-        privatizerClones.push_back(privatizerClone);
+        llvmPrivateVars.push_back(llvmReplacementValue);
+        mlirPrivatizerClones.push_back(mlirPrivatizerClone);
       }
 
       builder.restoreIP(oldIP);
@@ -1571,13 +1642,14 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       bodyGenStatus = failure();
 
     SmallVector<Region *> privateCleanupRegions;
-    llvm::transform(privatizerClones, std::back_inserter(privateCleanupRegions),
+    llvm::transform(mlirPrivatizerClones,
+                    std::back_inserter(privateCleanupRegions),
                     [](omp::PrivateClauseOp privatizer) {
                       return &privatizer.getDeallocRegion();
                     });
 
     if (failed(inlineOmpRegionCleanup(
-            privateCleanupRegions, privateVariables, moduleTranslation, builder,
+            privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder,
             "omp.private.dealloc", /*shouldLoadCleanupRegionArg=*/false)))
       bodyGenStatus = failure();
 
@@ -1604,7 +1676,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB,
                                  ifCond, numThreads, pbKind, isCancellable));
 
-  for (mlir::omp::PrivateClauseOp privatizerClone : privatizerClones)
+  for (mlir::omp::PrivateClauseOp privatizerClone : mlirPrivatizerClones)
     privatizerClone.erase();
 
   return bodyGenStatus;
diff --git a/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp b/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
index 3c8f64419d379ea..18a108be764b379 100644
--- a/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
+++ b/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
@@ -52,7 +52,7 @@ void spirv::appendModuleHeader(SmallVectorImpl<uint32_t> &header,
   // +-------------------------------------------------------------------------+
   header.push_back(spirv::kMagicNumber);
   header.push_back((majorVersion << 16) | (minorVersion << 8));
-  header.push_back(kGeneratorNumber);
+  header.push_back((kGeneratorNumber << 16) | LLVM_VERSION_MAJOR);
   header.push_back(idBound); // <id> bound
   header.push_back(0);       // Schema (reserved word)
 }
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 11e593cebc09b3d..bf2990c257bad2f 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2527,6 +2527,7 @@ LogicalResult OperationConverter::legalizeConvertedOpResultTypes(
           opReplacement->getConverter());
       rewriterImpl.mapping.map(result, castValue);
       inverseMapping[castValue].push_back(result);
+      llvm::erase(inverseMapping[newValue], result);
     }
   }
 
@@ -2680,7 +2681,7 @@ static void computeNecessaryMaterializations(
       if (!castOp)
         continue;
       if (castOp->getResultTypes() == inputOperands.getTypes()) {
-        replaceMaterialization(rewriterImpl, opResult, inputOperands,
+        replaceMaterialization(rewriterImpl, user->getResults(), inputOperands,
                                inverseMapping);
         necessaryMaterializations.remove(materializationOps.lookup(user));
       }
diff --git a/mlir/python/mlir/dialects/tensor.py b/mlir/python/mlir/dialects/tensor.py
index 79dd9476ad0ff91..0b30d1020990885 100644
--- a/mlir/python/mlir/dialects/tensor.py
+++ b/mlir/python/mlir/dialects/tensor.py
@@ -13,6 +13,7 @@
 
 from typing import Sequence, Union
 from ._ods_common import _cext as _ods_cext
+from ._ods_common import get_op_result_or_op_results as _get_op_result_or_op_results
 
 
 @_ods_cext.register_operation(_Dialect, replace=True)
@@ -43,6 +44,18 @@ def __init__(
         super().__init__(result_type, dynamic_sizes, loc=loc, ip=ip)
 
 
+def empty(
+    sizes: Sequence[Union[int, Value]],
+    element_type: Type,
+    *,
+    loc=None,
+    ip=None,
+) -> _ods_cext.ir.Value:
+    return _get_op_result_or_op_results(
+        EmptyOp(sizes=sizes, element_type=element_type, loc=loc, ip=ip)
+    )
+
+
 generate = region_op(
     lambda result, dynamic_extents: GenerateOp(result, dynamic_extents),
     terminator=lambda args: YieldOp(args[0]),
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir b/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir
new file mode 100644
index 000000000000000..14691e73e62d78c
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir
@@ -0,0 +1,139 @@
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx908 %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx90a %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx942 %s | FileCheck %s
+
+func.func @test_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @test_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 257, 10, 15, false : i32
+  // CHECK: return %0 : i32
+  %0 = amdgpu.dpp %arg0 %arg1 row_shl ( 0x1 : i32 ) { row_mask = 0xa : i32, bound_ctrl = false } : i32
+    return %0 : i32
+}
+
+func.func @quad_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @quad_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 145, 1, 1, true : i32
+  // CHECK: return %0 : i32
+  %0 = amdgpu.dpp %arg0 %arg1 quad_perm ( [1,0,1,2] ) { row_mask = 0x1 : i32, bank_mask = 0x1 : i32, bound_ctrl = true } : i32
+    return %0 : i32
+}
+
+func.func @wave_shr_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @wave_shr_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 312, 10, 1, true : i32
+  // CHECK: return %0 : i32
+  %0 = amdgpu.dpp %arg0 %arg1 wave_shr { row_mask = 0xa : i32, bank_mask = 0x1 : i32, bound_ctrl = true } : i32
+    return %0 : i32
+}
+
+func.func @row_half_mirror_update_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @row_half_mirror_update_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 321, 15, 1, false : i32
+  // CHECK: return %0 : i32
+%0 = amdgpu.dpp %arg0 %arg1 row_half_mirror { bank_mask = 0x1 : i32 } : i32
+    return %0 : i32
+}
+
+func.func @wave_rol_update_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @wave_rol_update_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 308, 10, 1, false : i32
+  // CHECK: return %0 : i32
+  %0 = amdgpu.dpp %arg0 %arg1 wave_rol { row_mask = 0xa : i32, bank_mask = 0x1 : i32 } : i32
+    return %0 : i32
+}
+
+func.func @row_bcast_dpp_f32(%arg0: f32, %arg1: f32) -> f32 {
+  // CHECK-LABEL: func @row_bcast_dpp_f32
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 322, 15, 15, true : f32
+  // CHECK: return %0 : f32
+  %0 = amdgpu.dpp %arg0 %arg1 row_bcast_15 { bound_ctrl = true } : f32
+    return %0 : f32
+}
+
+func.func @test_dpp_f32(%arg0: f32, %arg1: f32) -> f32 {
+  // CHECK-LABEL: func @test_dpp_f32
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 320, 1, 4, true : f32
+  // CHECK: return %0 : f32
+  %0 = amdgpu.dpp %arg0 %arg1 row_mirror { row_mask = 0x1 : i32, bank_mask = 0x4 : i32, bound_ctrl = true } : f32
+    return %0 : f32
+}
+
+func.func @quad_perm_update_dpp_f32(%arg0: f32, %arg1: f32) -> f32 {
+  // CHECK-LABEL: func @quad_perm_update_dpp_f32
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with  88, 15, 1, false : f32
+  // CHECK: return %0 : f32
+  %0 = amdgpu.dpp %arg0 %arg1 quad_perm ( [0,2,1,1] ) { bank_mask = 0x1 : i32 } : f32
+    return %0 : f32
+}
+
+func.func @quad_perm_dpp(%arg0: i64, %arg1: i64) -> i64 {
+  // CHECK-LABEL: func @quad_perm_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 88, 15, 15, false : i64
+  // CHECK: return %0 : i64
+  %0 = amdgpu.dpp %arg0 %arg1 quad_perm ( [0,2,1,1] ) : i64
+    return %0 : i64
+}
+
+func.func @row_bcast_dpp(%arg0: f64, %arg1: f64) -> f64 {
+  // CHECK-LABEL: func @row_bcast_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 323, 4, 1, false : f64
+  // CHECK: return %0 : f64
+  %0 = amdgpu.dpp %arg0 %arg1 row_bcast_31 { row_mask = 0x4 : i32, bank_mask = 0x1 : i32} : f64
+    return %0 : f64
+}
+
+func.func @test_dpp_f16(%arg0: f16, %arg1: f16) -> f16 {
+  // CHECK-LABEL:  func @test_dpp_f16
+  // CHECK: llvm.bitcast %arg1 : f16 to i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %0, %1[%2 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %3 : vector<2xi16> to i32
+  // CHECK: llvm.bitcast %arg0 : f16 to i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %5, %6[%7 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %8 : vector<2xi16> to i32
+  // CHECK: rocdl.update.dpp %9, %4 with 273, 15, 3, false : i32
+  // CHECK: llvm.trunc %10 : i32 to i16
+  // CHECK: llvm.bitcast %11 : i16 to f16
+  // CHECK: return %12 : f16
+  %0 = amdgpu.dpp %arg0 %arg1 row_shr ( 0x1 : i32 ){ bank_mask = 0x3 : i32 } : f16
+    return %0 : f16
+}
+
+func.func @row_shl_dpp_i16(%arg0: i16, %arg1: i16) -> i16 {
+  // CHECK-LABEL: func @row_shl_dpp_i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %arg1, %0[%1 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %2 : vector<2xi16> to i32
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %arg0, %4[%5 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %6 : vector<2xi16> to i32
+  // CHECK: rocdl.update.dpp %7, %3 with 298, 10, 1, false : i32
+  // CHECK: llvm.trunc %8 : i32 to i16
+  // CHECK: return %9 : i16
+  %0 = amdgpu.dpp %arg0 %arg1 row_ror ( 0xa : i32 ) { row_mask = 0xa : i32, bank_mask = 0x1 : i32 } : i16
+    return %0 : i16
+}
+
+func.func @row_bcast_update_dpp_f16(%arg0: f16, %arg1: f16) -> f16 {
+  // CHECK-LABEL: func @row_bcast_update_dpp_f16
+  // CHECK: llvm.bitcast %arg1 : f16 to i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %0, %1[%2 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %arg0 : f16 to i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK  llvm.insertelement %5, %6[%7 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %8 : vector<2xi16> to i32
+  // CHECK: rocdl.update.dpp %9, %4 with 322, 15, 15, true : i32
+  // CHECK: llvm.trunc %10 : i32 to i16
+  // CHECK: llvm.bitcast %11 : i16 to f16
+  // CHECK: return %12 : f16
+  %0 = amdgpu.dpp %arg0 %arg1 row_bcast_15 { bound_ctrl = true } : f16
+    return %0 : f16
+}
diff --git a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
index 8e133288b832b68..ec4f4a304d50734 100644
--- a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
+++ b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
@@ -444,7 +444,7 @@ gpu.module @kernels {
 // CHECK:           %[[VAL_17:.*]] = llvm.insertvalue %[[VAL_15]], %[[VAL_16]][0]
 // CHECK:           llvm.insertvalue %[[VAL_15]], %[[VAL_17]][1]
   gpu.func @kernel_with_private_attributions()
-      private(%arg2: memref<32xf32>, %arg3: memref<16xi16>)
+      private(%arg2: memref<32xf32, #gpu.address_space<private>>, %arg3: memref<16xi16, #gpu.address_space<private>>)
       kernel {
     gpu.return
   }
@@ -471,7 +471,7 @@ gpu.module @kernels {
 // CHECK:           %[[VAL_42:.*]] = llvm.insertvalue %[[VAL_30]], %[[VAL_41]][0]
 // CHECK:           llvm.insertvalue %[[VAL_30]], %[[VAL_42]][1]
   gpu.func @kernel_with_workgoup_attributions()
-      workgroup(%arg2: memref<32xf32, 3>, %arg3: memref<16xi16, 3>)
+      workgroup(%arg2: memref<32xf32, #gpu.address_space<workgroup>>, %arg3: memref<16xi16, #gpu.address_space<workgroup>>)
       kernel {
     gpu.return
   }
@@ -491,8 +491,8 @@ gpu.module @kernels {
 // CHECK-64:        %[[VAL_92:.*]] = llvm.alloca %[[VAL_91]] x i64 : (i64) -> !llvm.ptr
 // CHECK-32:        %[[VAL_92:.*]] = llvm.alloca %[[VAL_91]] x i32 : (i64) -> !llvm.ptr
   gpu.func @kernel_with_both_attributions()
-      workgroup(%arg4: memref<8xf32, 3>, %arg5: memref<16xindex, 3>)
-      private(%arg6: memref<32xi32>, %arg7: memref<32xindex>)
+      workgroup(%arg4: memref<8xf32, #gpu.address_space<workgroup>>, %arg5: memref<16xindex, #gpu.address_space<workgroup>>)
+      private(%arg6: memref<32xi32, #gpu.address_space<private>>, %arg7: memref<32xindex, #gpu.address_space<private>>)
       kernel {
     gpu.return
   }
@@ -503,3 +503,15 @@ gpu.module @kernels {
     gpu.return
   }
 }
+
+// -----
+
+gpu.module @kernels {
+// CHECK-LABEL:   llvm.func spir_funccc @address_spaces(
+// CHECK-SAME:                                          {{.*}}: !llvm.ptr<1>
+// CHECK-SAME:                                          {{.*}}: !llvm.ptr<3>
+// CHECK-SAME:                                          {{.*}}: !llvm.ptr
+  gpu.func @address_spaces(%arg0: memref<f32, #gpu.address_space<global>>, %arg1: memref<f32, #gpu.address_space<workgroup>>, %arg2: memref<f32, #gpu.address_space<private>>) {
+    gpu.return
+  }
+}
diff --git a/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir b/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
index a93976673934295..ecbd59e54971ef4 100644
--- a/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
+++ b/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
@@ -22,22 +22,30 @@ func.func @float32_unary_scalar(%arg0: f32) {
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.GL.Log %[[ADDONE]]
   %5 = math.log1p %arg0 : f32
+  // CHECK: %[[LOG2_RECIPROCAL:.+]] = spirv.Constant 1.44269502 : f32
+  // CHECK: %[[LOG0:.+]] = spirv.GL.Log {{.+}}
+  // CHECK: spirv.FMul %[[LOG0]], %[[LOG2_RECIPROCAL]]
+  %6 = math.log2 %arg0 : f32
+  // CHECK: %[[LOG10_RECIPROCAL:.+]] = spirv.Constant 0.434294492 : f32
+  // CHECK: %[[LOG1:.+]] = spirv.GL.Log {{.+}}
+  // CHECK: spirv.FMul %[[LOG1]], %[[LOG10_RECIPROCAL]]
+  %7 = math.log10 %arg0 : f32
   // CHECK: spirv.GL.RoundEven %{{.*}}: f32
-  %6 = math.roundeven %arg0 : f32
+  %8 = math.roundeven %arg0 : f32
   // CHECK: spirv.GL.InverseSqrt %{{.*}}: f32
-  %7 = math.rsqrt %arg0 : f32
+  %9 = math.rsqrt %arg0 : f32
   // CHECK: spirv.GL.Sqrt %{{.*}}: f32
-  %8 = math.sqrt %arg0 : f32
+  %10 = math.sqrt %arg0 : f32
   // CHECK: spirv.GL.Tanh %{{.*}}: f32
-  %9 = math.tanh %arg0 : f32
+  %11 = math.tanh %arg0 : f32
   // CHECK: spirv.GL.Sin %{{.*}}: f32
-  %10 = math.sin %arg0 : f32
+  %12 = math.sin %arg0 : f32
   // CHECK: spirv.GL.FAbs %{{.*}}: f32
-  %11 = math.absf %arg0 : f32
+  %13 = math.absf %arg0 : f32
   // CHECK: spirv.GL.Ceil %{{.*}}: f32
-  %12 = math.ceil %arg0 : f32
+  %14 = math.ceil %arg0 : f32
   // CHECK: spirv.GL.Floor %{{.*}}: f32
-  %13 = math.floor %arg0 : f32
+  %15 = math.floor %arg0 : f32
   return
 }
 
@@ -59,16 +67,24 @@ func.func @float32_unary_vector(%arg0: vector<3xf32>) {
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.GL.Log %[[ADDONE]]
   %5 = math.log1p %arg0 : vector<3xf32>
+  // CHECK: %[[LOG2_RECIPROCAL:.+]] = spirv.Constant dense<1.44269502> : vector<3xf32>
+  // CHECK: %[[LOG0:.+]] = spirv.GL.Log {{.+}}
+  // CHECK: spirv.FMul %[[LOG0]], %[[LOG2_RECIPROCAL]]
+  %6 = math.log2 %arg0 : vector<3xf32>
+  // CHECK: %[[LOG10_RECIPROCAL:.+]] = spirv.Constant dense<0.434294492> : vector<3xf32>
+  // CHECK: %[[LOG1:.+]] = spirv.GL.Log {{.+}}
+  // CHECK: spirv.FMul %[[LOG1]], %[[LOG10_RECIPROCAL]]
+  %7 = math.log10 %arg0 : vector<3xf32>
   // CHECK: spirv.GL.RoundEven %{{.*}}: vector<3xf32>
-  %6 = math.roundeven %arg0 : vector<3xf32>
+  %8 = math.roundeven %arg0 : vector<3xf32>
   // CHECK: spirv.GL.InverseSqrt %{{.*}}: vector<3xf32>
-  %7 = math.rsqrt %arg0 : vector<3xf32>
+  %9 = math.rsqrt %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Sqrt %{{.*}}: vector<3xf32>
-  %8 = math.sqrt %arg0 : vector<3xf32>
+  %10 = math.sqrt %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Tanh %{{.*}}: vector<3xf32>
-  %9 = math.tanh %arg0 : vector<3xf32>
+  %11 = math.tanh %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Sin %{{.*}}: vector<3xf32>
-  %10 = math.sin %arg0 : vector<3xf32>
+  %12 = math.sin %arg0 : vector<3xf32>
   return
 }
 
diff --git a/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir b/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
index e9ca838354c0de5..393a910c1fb1d74 100644
--- a/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
+++ b/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
@@ -20,26 +20,34 @@ func.func @float32_unary_scalar(%arg0: f32) {
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.CL.log %[[ADDONE]]
   %5 = math.log1p %arg0 : f32
+  // CHECK: %[[LOG2_RECIPROCAL:.+]] = spirv.Constant 1.44269502 : f32
+  // CHECK: %[[LOG0:.+]] = spirv.CL.log {{.+}}
+  // CHECK: spirv.FMul %[[LOG0]], %[[LOG2_RECIPROCAL]]
+  %6 = math.log2 %arg0 : f32
+  // CHECK: %[[LOG10_RECIPROCAL:.+]] = spirv.Constant 0.434294492 : f32
+  // CHECK: %[[LOG1:.+]] = spirv.CL.log {{.+}}
+  // CHECK: spirv.FMul %[[LOG1]], %[[LOG10_RECIPROCAL]]
+  %7 = math.log10 %arg0 : f32
   // CHECK: spirv.CL.rint %{{.*}}: f32
-  %6 = math.roundeven %arg0 : f32
+  %8 = math.roundeven %arg0 : f32
   // CHECK: spirv.CL.rsqrt %{{.*}}: f32
-  %7 = math.rsqrt %arg0 : f32
+  %9 = math.rsqrt %arg0 : f32
   // CHECK: spirv.CL.sqrt %{{.*}}: f32
-  %8 = math.sqrt %arg0 : f32
+  %10 = math.sqrt %arg0 : f32
   // CHECK: spirv.CL.tanh %{{.*}}: f32
-  %9 = math.tanh %arg0 : f32
+  %11 = math.tanh %arg0 : f32
   // CHECK: spirv.CL.sin %{{.*}}: f32
-  %10 = math.sin %arg0 : f32
+  %12 = math.sin %arg0 : f32
   // CHECK: spirv.CL.fabs %{{.*}}: f32
-  %11 = math.absf %arg0 : f32
+  %13 = math.absf %arg0 : f32
   // CHECK: spirv.CL.ceil %{{.*}}: f32
-  %12 = math.ceil %arg0 : f32
+  %14 = math.ceil %arg0 : f32
   // CHECK: spirv.CL.floor %{{.*}}: f32
-  %13 = math.floor %arg0 : f32
+  %15 = math.floor %arg0 : f32
   // CHECK: spirv.CL.erf %{{.*}}: f32
-  %14 = math.erf %arg0 : f32
+  %16 = math.erf %arg0 : f32
   // CHECK: spirv.CL.round %{{.*}}: f32
-  %15 = math.round %arg0 : f32
+  %17 = math.round %arg0 : f32
   return
 }
 
@@ -61,16 +69,24 @@ func.func @float32_unary_vector(%arg0: vector<3xf32>) {
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.CL.log %[[ADDONE]]
   %5 = math.log1p %arg0 : vector<3xf32>
+  // CHECK: %[[LOG2_RECIPROCAL:.+]] = spirv.Constant dense<1.44269502> : vector<3xf32>
+  // CHECK: %[[LOG0:.+]] = spirv.CL.log {{.+}}
+  // CHECK: spirv.FMul %[[LOG0]], %[[LOG2_RECIPROCAL]]
+  %6 = math.log2 %arg0 : vector<3xf32>
+  // CHECK: %[[LOG10_RECIPROCAL:.+]] = spirv.Constant dense<0.434294492> : vector<3xf32>
+  // CHECK: %[[LOG1:.+]] = spirv.CL.log {{.+}}
+  // CHECK: spirv.FMul %[[LOG1]], %[[LOG10_RECIPROCAL]]
+  %7 = math.log10 %arg0 : vector<3xf32>
   // CHECK: spirv.CL.rint %{{.*}}: vector<3xf32>
-  %6 = math.roundeven %arg0 : vector<3xf32>
+  %8 = math.roundeven %arg0 : vector<3xf32>
   // CHECK: spirv.CL.rsqrt %{{.*}}: vector<3xf32>
-  %7 = math.rsqrt %arg0 : vector<3xf32>
+  %9 = math.rsqrt %arg0 : vector<3xf32>
   // CHECK: spirv.CL.sqrt %{{.*}}: vector<3xf32>
-  %8 = math.sqrt %arg0 : vector<3xf32>
+  %10 = math.sqrt %arg0 : vector<3xf32>
   // CHECK: spirv.CL.tanh %{{.*}}: vector<3xf32>
-  %9 = math.tanh %arg0 : vector<3xf32>
+  %11 = math.tanh %arg0 : vector<3xf32>
   // CHECK: spirv.CL.sin %{{.*}}: vector<3xf32>
-  %10 = math.sin %arg0 : vector<3xf32>
+  %12 = math.sin %arg0 : vector<3xf32>
   return
 }
 
diff --git a/mlir/test/Conversion/SCFToEmitC/switch.mlir b/mlir/test/Conversion/SCFToEmitC/switch.mlir
new file mode 100644
index 000000000000000..659d9f43963efa5
--- /dev/null
+++ b/mlir/test/Conversion/SCFToEmitC/switch.mlir
@@ -0,0 +1,117 @@
+// RUN: mlir-opt -allow-unregistered-dialect -convert-scf-to-emitc %s | FileCheck %s
+
+// CHECK-LABEL:   func.func @switch_no_result(
+// CHECK-SAME:                                %[[VAL_0:.*]]: index) {
+// CHECK:           emitc.switch %[[VAL_0]]
+// CHECK:           case 2 {
+// CHECK:             %[[VAL_1:.*]] = arith.constant 10 : i32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           case 5 {
+// CHECK:             %[[VAL_2:.*]] = arith.constant 20 : i32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           default {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 30 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @switch_no_result(%arg0 : index) {
+    scf.index_switch %arg0
+    case 2 {
+      %1 = arith.constant 10 : i32
+      scf.yield
+    }
+    case 5 {
+      %2 = arith.constant 20 : i32
+      scf.yield
+    }
+    default {
+      %3 = arith.constant 30 : i32
+    }
+  return
+}
+
+// CHECK-LABEL:   func.func @switch_one_result(
+// CHECK-SAME:                                 %[[VAL_0:.*]]: index) {
+// CHECK:           %[[VAL_1:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32
+// CHECK:           emitc.switch %[[VAL_0]]
+// CHECK:           case 2 {
+// CHECK:             %[[VAL_2:.*]] = arith.constant 10 : i32
+// CHECK:             emitc.assign %[[VAL_2]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           case 5 {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 20 : i32
+// CHECK:             emitc.assign %[[VAL_3]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           default {
+// CHECK:             %[[VAL_4:.*]] = arith.constant 30 : i32
+// CHECK:             emitc.assign %[[VAL_4]] : i32 to %[[VAL_1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @switch_one_result(%arg0 : index) {
+    %0 = scf.index_switch %arg0 -> i32
+    case 2 {
+      %1 = arith.constant 10 : i32
+      scf.yield %1 : i32
+    }
+    case 5 {
+      %2 = arith.constant 20 : i32
+      scf.yield %2 : i32
+    }
+    default {
+      %3 = arith.constant 30 : i32
+      scf.yield %3 : i32
+    }
+  return
+}
+
+// CHECK-LABEL:   func.func @switch_two_results(
+// CHECK-SAME:                                  %[[VAL_0:.*]]: index) {
+// CHECK:           %[[VAL_1:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32
+// CHECK:           %[[VAL_2:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
+// CHECK:           emitc.switch %[[VAL_0]]
+// CHECK:           case 2 {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 10 : i32
+// CHECK:             %[[VAL_4:.*]] = arith.constant 1.200000e+00 : f32
+// CHECK:             emitc.assign %[[VAL_3]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.assign %[[VAL_4]] : f32 to %[[VAL_2]] : f32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           case 5 {
+// CHECK:             %[[VAL_5:.*]] = arith.constant 20 : i32
+// CHECK:             %[[VAL_6:.*]] = arith.constant 2.400000e+00 : f32
+// CHECK:             emitc.assign %[[VAL_5]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.assign %[[VAL_6]] : f32 to %[[VAL_2]] : f32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           default {
+// CHECK:             %[[VAL_7:.*]] = arith.constant 30 : i32
+// CHECK:             %[[VAL_8:.*]] = arith.constant 3.600000e+00 : f32
+// CHECK:             emitc.assign %[[VAL_7]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.assign %[[VAL_8]] : f32 to %[[VAL_2]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @switch_two_results(%arg0 : index) {
+    %0, %1 = scf.index_switch %arg0 -> i32, f32
+    case 2 {
+      %2 = arith.constant 10 : i32
+      %3 = arith.constant 1.2 : f32
+      scf.yield %2, %3 : i32, f32
+    }
+    case 5 {
+      %4 = arith.constant 20 : i32
+      %5 = arith.constant 2.4 : f32
+      scf.yield %4, %5 : i32, f32
+    }
+    default {
+      %6 = arith.constant 30 : i32
+      %7 = arith.constant 3.6 : f32
+      scf.yield %6, %7 : i32, f32
+    }
+  return
+}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index 9b61c4493994c2e..0cd7ee6fb424fc0 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -1245,9 +1245,9 @@ func.func @extract_element_with_value_2d(%arg0: vector<1x16xf32>, %arg1: index)
 
 // -----
 
-// CHECK-LABEL: @insert_element_0d
+// CHECK-LABEL: @insertelement_0d
 // CHECK-SAME: %[[A:.*]]: f32,
-func.func @insert_element_0d(%a: f32, %b: vector<f32>) -> vector<f32> {
+func.func @insertelement_0d(%a: f32, %b: vector<f32>) -> vector<f32> {
   // CHECK: %[[B:.*]] =  builtin.unrealized_conversion_cast %{{.*}} :
   // CHECK:   vector<f32> to vector<1xf32>
   // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -1258,26 +1258,38 @@ func.func @insert_element_0d(%a: f32, %b: vector<f32>) -> vector<f32> {
 
 // -----
 
-func.func @insert_element(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
+func.func @insertelement(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
   %0 = arith.constant 3 : i32
   %1 = vector.insertelement %arg0, %arg1[%0 : i32] : vector<4xf32>
   return %1 : vector<4xf32>
 }
-// CHECK-LABEL: @insert_element(
+// CHECK-LABEL: @insertelement(
 // CHECK-SAME: %[[A:.*]]: f32,
 // CHECK-SAME: %[[B:.*]]: vector<4xf32>)
 //       CHECK:   %[[c:.*]] = arith.constant 3 : i32
 //       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[c]] : i32] : vector<4xf32>
 //       CHECK:   return %[[x]] : vector<4xf32>
 
+func.func @insertelement_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = arith.constant 3 : i32
+  %1 = vector.insertelement %arg0, %arg1[%0 : i32] : vector<[4]xf32>
+  return %1 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insertelement_scalable(
+// CHECK-SAME: %[[A:.*]]: f32,
+// CHECK-SAME: %[[B:.*]]: vector<[4]xf32>)
+//       CHECK:   %[[c:.*]] = arith.constant 3 : i32
+//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[c]] : i32] : vector<[4]xf32>
+//       CHECK:   return %[[x]] : vector<[4]xf32>
+
 // -----
 
-func.func @insert_element_index(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
+func.func @insertelement_index(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
   %0 = arith.constant 3 : index
   %1 = vector.insertelement %arg0, %arg1[%0 : index] : vector<4xf32>
   return %1 : vector<4xf32>
 }
-// CHECK-LABEL: @insert_element_index(
+// CHECK-LABEL: @insertelement_index(
 //  CHECK-SAME: %[[A:.*]]: f32,
 //  CHECK-SAME: %[[B:.*]]: vector<4xf32>)
 //       CHECK:   %[[c:.*]] = arith.constant 3 : index
@@ -1285,6 +1297,19 @@ func.func @insert_element_index(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf3
 //       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[i]] : i64] : vector<4xf32>
 //       CHECK:   return %[[x]] : vector<4xf32>
 
+func.func @insertelement_index_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = arith.constant 3 : index
+  %1 = vector.insertelement %arg0, %arg1[%0 : index] : vector<[4]xf32>
+  return %1 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insertelement_index_scalable(
+//  CHECK-SAME: %[[A:.*]]: f32,
+//  CHECK-SAME: %[[B:.*]]: vector<[4]xf32>)
+//       CHECK:   %[[c:.*]] = arith.constant 3 : index
+//       CHECK:   %[[i:.*]] = builtin.unrealized_conversion_cast %[[c]] : index to i64
+//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[i]] : i64] : vector<[4]xf32>
+//       CHECK:   return %[[x]] : vector<[4]xf32>
+
 // -----
 
 func.func @insert_element_into_vec_1d(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
@@ -1296,6 +1321,15 @@ func.func @insert_element_into_vec_1d(%arg0: f32, %arg1: vector<4xf32>) -> vecto
 //       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<4xf32>
 //       CHECK:   return {{.*}} : vector<4xf32>
 
+func.func @insert_element_into_vec_1d_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = vector.insert %arg0, %arg1[3] : f32 into vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insert_element_into_vec_1d_scalable
+//       CHECK:   llvm.mlir.constant(3 : i64) : i64
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<[4]xf32>
+//       CHECK:   return {{.*}} : vector<[4]xf32>
+
 // -----
 
 func.func @insert_index_element_into_vec_1d(%arg0: index, %arg1: vector<4xindex>) -> vector<4xindex> {
@@ -1312,6 +1346,21 @@ func.func @insert_index_element_into_vec_1d(%arg0: index, %arg1: vector<4xindex>
 //       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<4xi64> to vector<4xindex>
 //       CHECK:   return %[[T5]] : vector<4xindex>
 
+
+func.func @insert_index_element_into_vec_1d_scalable(%arg0: index, %arg1: vector<[4]xindex>) -> vector<[4]xindex> {
+  %0 = vector.insert %arg0, %arg1[3] : index into vector<[4]xindex>
+  return %0 : vector<[4]xindex>
+}
+// CHECK-LABEL: @insert_index_element_into_vec_1d_scalable(
+// CHECK-SAME: %[[A:.*]]: index,
+// CHECK-SAME: %[[B:.*]]: vector<[4]xindex>)
+//       CHECK-DAG:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : index to i64
+//       CHECK-DAG:   %[[T1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<[4]xindex> to vector<[4]xi64>
+//       CHECK:   %[[T3:.*]] = llvm.mlir.constant(3 : i64) : i64
+//       CHECK:   %[[T4:.*]] = llvm.insertelement %[[T0]], %[[T1]][%[[T3]] : i64] : vector<[4]xi64>
+//       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<[4]xi64> to vector<[4]xindex>
+//       CHECK:   return %[[T5]] : vector<[4]xindex>
+
 // -----
 
 func.func @insert_vec_2d_into_vec_3d(%arg0: vector<8x16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
@@ -1322,6 +1371,14 @@ func.func @insert_vec_2d_into_vec_3d(%arg0: vector<8x16xf32>, %arg1: vector<4x8x
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+func.func @insert_vec_2d_into_vec_3d_scalable(%arg0: vector<8x[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3] : vector<8x[16]xf32> into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_vec_2d_into_vec_3d_scalable
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
 // -----
 
 func.func @insert_vec_1d_into_vec_3d(%arg0: vector<16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
@@ -1332,6 +1389,14 @@ func.func @insert_vec_1d_into_vec_3d(%arg0: vector<16xf32>, %arg1: vector<4x8x16
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+func.func @insert_vec_1d_into_vec_3d_scalable(%arg0: vector<[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3, 7] : vector<[16]xf32> into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_vec_1d_into_vec_3d_scalable
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
 // -----
 
 func.func @insert_element_into_vec_3d(%arg0: f32, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
@@ -1345,6 +1410,17 @@ func.func @insert_element_into_vec_3d(%arg0: f32, %arg1: vector<4x8x16xf32>) ->
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+func.func @insert_element_into_vec_3d_scalable(%arg0: f32, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3, 7, 15] : f32 into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_element_into_vec_3d_scalable
+//       CHECK:   llvm.extractvalue {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<[16]xf32>
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
 // -----
 
 func.func @insert_element_with_value_1d(%arg0: vector<16xf32>, %arg1: f32, %arg2: index)
@@ -1358,6 +1434,17 @@ func.func @insert_element_with_value_1d(%arg0: vector<16xf32>, %arg1: f32, %arg2
 //       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
 //       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<16xf32>
 
+func.func @insert_element_with_value_1d_scalable(%arg0: vector<[16]xf32>, %arg1: f32, %arg2: index)
+                                      -> vector<[16]xf32> {
+  %0 = vector.insert %arg1, %arg0[%arg2]: f32 into vector<[16]xf32>
+  return %0 : vector<[16]xf32>
+}
+
+// CHECK-LABEL: @insert_element_with_value_1d_scalable
+//  CHECK-SAME:   %[[DST:.+]]: vector<[16]xf32>, %[[SRC:.+]]: f32, %[[INDEX:.+]]: index
+//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
+//       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<[16]xf32>
+
 // -----
 
 func.func @insert_element_with_value_2d(%base: vector<1x16xf32>, %value: f32, %idx: index)
diff --git a/mlir/test/Dialect/EmitC/invalid_ops.mlir b/mlir/test/Dialect/EmitC/invalid_ops.mlir
index 4181b726593e4a5..4b5bcf46c1aab97 100644
--- a/mlir/test/Dialect/EmitC/invalid_ops.mlir
+++ b/mlir/test/Dialect/EmitC/invalid_ops.mlir
@@ -227,7 +227,7 @@ func.func @sub_pointer_pointer(%arg0: !emitc.ptr<f32>, %arg1: !emitc.ptr<f32>) {
 // -----
 
 func.func @test_misplaced_yield() {
-  // expected-error @+1 {{'emitc.yield' op expects parent op to be one of 'emitc.expression, emitc.if, emitc.for'}}
+  // expected-error @+1 {{'emitc.yield' op expects parent op to be one of 'emitc.expression, emitc.if, emitc.for, emitc.switch'}}
   emitc.yield
   return
 }
@@ -466,3 +466,89 @@ func.func @member_of_ptr(%arg0: i32) {
   %0 = "emitc.member_of_ptr" (%arg0) {member = "a"} : (i32) -> i32
   return
 }
+
+// -----
+
+func.func @emitc_switch() {
+  %0 = "emitc.variable"(){value = 1 : i16} : () -> i16
+
+  // expected-error@+1 {{'emitc.switch' op expected region to end with emitc.yield, but got emitc.call_opaque}}
+  emitc.switch %0 : i16
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// -----
+
+func.func @emitc_switch() {
+  %0 = "emitc.variable"(){value = 1 : i32} : () -> i32
+
+  emitc.switch %0 : i32
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  // expected-error@+1 {{custom op 'emitc.switch' expected integer value}}
+  case {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// -----
+
+func.func @emitc_switch() {
+  %0 = "emitc.variable"(){value = 1 : i8} : () -> i8
+
+  emitc.switch %0 : i8
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 3 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  // expected-error@+1 {{custom op 'emitc.switch' expected 'default'}}
+  return
+}
+
+// -----
+
+func.func @emitc_switch() {
+  %0 = "emitc.variable"(){value = 1 : i64} : () -> i64
+
+  // expected-error@+1 {{'emitc.switch' op has duplicate case value: 2}}
+  emitc.switch %0 : i64
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 2 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
diff --git a/mlir/test/Dialect/EmitC/ops.mlir b/mlir/test/Dialect/EmitC/ops.mlir
index c6d7dd1fd004a4c..aa6f043e66cdede 100644
--- a/mlir/test/Dialect/EmitC/ops.mlir
+++ b/mlir/test/Dialect/EmitC/ops.mlir
@@ -261,3 +261,23 @@ func.func @member_access(%arg0: !emitc.opaque<"mystruct">, %arg1: !emitc.opaque<
   %2 = "emitc.member_of_ptr" (%arg2) {member = "a"} : (!emitc.ptr<!emitc.opaque<"mystruct">>) -> i32
   return
 }
+
+func.func @switch() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> !emitc.ptrdiff_t
+
+  emitc.switch %0 : !emitc.ptrdiff_t
+  case 1 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 2 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+  }
+
+  return 
+}
diff --git a/mlir/test/Dialect/GPU/subgroup-redule-lowering.mlir b/mlir/test/Dialect/GPU/subgroup-reduce-lowering.mlir
similarity index 100%
rename from mlir/test/Dialect/GPU/subgroup-redule-lowering.mlir
rename to mlir/test/Dialect/GPU/subgroup-reduce-lowering.mlir
diff --git a/mlir/test/Dialect/LLVMIR/inlining-alias-scopes.mlir b/mlir/test/Dialect/LLVMIR/inlining-alias-scopes.mlir
index a91b991c5ed2b92..bd5e7aa996ada7e 100644
--- a/mlir/test/Dialect/LLVMIR/inlining-alias-scopes.mlir
+++ b/mlir/test/Dialect/LLVMIR/inlining-alias-scopes.mlir
@@ -234,6 +234,29 @@ llvm.func @bar(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr) {
 
 // -----
 
+// CHECK-DAG: #[[DOMAIN:.*]] = #llvm.alias_scope_domain<{{.*}}>
+// CHECK-DAG: #[[$ARG_SCOPE:.*]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]]{{(,.*)?}}>
+
+llvm.func @foo(%arg0: !llvm.ptr {llvm.noalias}, %arg1: !llvm.ptr) {
+  %0 = llvm.mlir.constant(5 : i64) : i64
+  %1 = llvm.load %arg0 {alignment = 4 : i64} : !llvm.ptr -> f32
+  %2 = llvm.getelementptr inbounds %arg1[%0] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+  llvm.store %1, %2 {alignment = 4 : i64} : f32, !llvm.ptr
+  llvm.return
+}
+
+// CHECK-LABEL: llvm.func @missing_noalias_on_one_ptr
+// CHECK: llvm.load
+// CHECK-SAME: alias_scopes = [#[[$ARG_SCOPE]]]
+// CHECK: llvm.store
+// CHECK-SAME: noalias_scopes = [#[[$ARG_SCOPE]]]
+llvm.func @missing_noalias_on_one_ptr(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr) {
+  llvm.call @foo(%arg0, %arg2) : (!llvm.ptr, !llvm.ptr) -> ()
+  llvm.return
+}
+
+// -----
+
 // CHECK-DAG: #[[DOMAIN:.*]] = #llvm.alias_scope_domain<{{.*}}>
 // CHECK-DAG: #[[$ARG0_SCOPE:.*]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]]{{(,.*)?}}>
 // CHECK-DAG: #[[$ARG1_SCOPE:.*]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]]{{(,.*)?}}>
diff --git a/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
new file mode 100644
index 000000000000000..6bb3fb1423edc62
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
@@ -0,0 +1,292 @@
+// RUN: mlir-opt %s -transform-interpreter -canonicalize --split-input-file | FileCheck %s
+
+func.func @conv2d(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
+  %0 = tensor.empty() : tensor<6x6x5x2xf32>
+  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %2 = tensor.empty() : tensor<6x6x2x2x2x5xf32>
+  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%2 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
+  %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x6x2x2x2x5xf32> into tensor<36x8x5xf32>
+  %4 = tensor.empty() : tensor<36x8x2xf32>
+  %5 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<36x8x5xf32>, tensor<36x5x2xf32>) outs(%4 : tensor<36x8x2xf32>) -> tensor<36x8x2xf32>
+  %expanded = tensor.expand_shape %5 [[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 2, 2, 2, 2] : tensor<36x8x2xf32> into tensor<6x6x2x2x2x2xf32>
+  %6 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x6x2x2x2x2xf32>) outs(%arg2 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
+  return %6 : tensor<2x8x8x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %3, %loop3:2 = transform.structured.tile_using_for %2 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %4 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %5, %loop5:2 = transform.structured.tile_using_for %4 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %7 = transform.structured.decompose_winograd_op %0 : (!transform.any_op) -> (!transform.any_op)
+    %8 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %3 : (!transform.any_op) -> !transform.any_op
+    %9 = transform.structured.decompose_winograd_op %8 : (!transform.any_op) -> (!transform.any_op)
+    %10 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %5 : (!transform.any_op) -> !transform.any_op
+    %11 = transform.structured.decompose_winograd_op %10 : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1) -> ()>
+// CHECK: #[[$MAP2:.+]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK-LABEL: func.func @conv2d
+// CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<2x3x3x5xf32>, %[[ARG2:.*]]: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
+// CHECK:  %[[CST:.*]] = arith.constant 1.024000e+03 : f32
+// CHECK:  %[[C1:.*]] = arith.constant 1 : index
+// CHECK:  %[[C5:.*]] = arith.constant 5 : index
+// CHECK:  %[[C2:.*]] = arith.constant 2 : index
+// CHECK:  %[[C0:.*]] = arith.constant 0 : index
+// CHECK:  %[[S0:.*]] = tensor.empty()
+// CHECK:  %[[S1:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S0]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 3, 3, 1] [1, 1, 1, 1]
+// CHECK:      %[[S11:.*]] = linalg.matmul
+// CHECK:      %[[S13:.*]] = linalg.matmul
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S13]] into %[[ARG6]][0, 0, %[[ARG5]], %[[ARG3]]] [6, 6, 1, 1] [1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+// CHECK:  %[[S2:.*]] = tensor.empty() : tensor<6x6x2x2x2x5xf32>
+// CHECK:  %[[S3:.*]] = tensor.empty() : tensor<6x6x2x2x2x5xf32>
+// CHECK:  %[[S4:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S3]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
+// CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
+// CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, %[[S10]], %[[S11]], 0] [2, 6, 6, 5] [1, 1, 1, 1]
+// CHECK:      %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[S2]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[S12:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[EXTRACTED_SLICE_7]])
+// CHECK:        %[[S13:.*]] = scf.for %[[ARG9:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG10:.*]] = %[[ARG8]])
+// CHECK:          %[[EXTRACTED_SLICE_8:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE]][%[[ARG7]], 0, 0, %[[ARG9]]] [1, 6, 6, 1] [1, 1, 1, 1]
+// CHECK:          %[[S15:.*]] = linalg.matmul
+// CHECK:          %[[S17:.*]] = linalg.matmul
+// CHECK:          %[[INSERTED_SLICE_9:.*]] = tensor.insert_slice %[[S17]] into %[[ARG10]][0, 0, 0, 0, %[[ARG7]], %[[ARG9]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          scf.yield %[[INSERTED_SLICE_9]]
+// CHECK:        scf.yield %[[S13]]
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG6]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+// CHECK:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
+// CHECK:  %[[COLLAPSED_6:.*]] = tensor.collapse_shape %[[S4]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:  %[[S6:.*]] = linalg.batch_matmul
+// CHECK:  %[[EXPANDED:.*]] = tensor.expand_shape %[[S6]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 2, 2, 2, 2]
+// CHECK:  %[[S7:.*]] = tensor.empty() : tensor<2x8x8x2xf32>
+// CHECK:  %[[S8:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S7]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
+// CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
+// CHECK:      %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[ARG2]][0, %[[S10]], %[[S11]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      %[[S12:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[EXTRACTED_SLICE_7]])
+// CHECK:        %[[S15:.*]] = scf.for %[[ARG9:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG10:.*]] = %[[ARG8]])
+// CHECK:          %[[EXTRACTED_SLICE_8:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE]][0, 0, 0, 0, %[[ARG7]], %[[ARG9]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          %[[S17:.*]] = linalg.matmul
+// CHECK:          %[[S19:.*]] = linalg.matmul
+// CHECK:          %[[S20:.*]] = tensor.empty()
+// CHECK:          %[[S21:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S20]] : tensor<4x4xf32>) {
+// CHECK:          ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:            linalg.yield %[[IN]] : f32
+// CHECK:          } -> tensor<4x4xf32>
+// CHECK:          %[[S22:.*]] = linalg.mul ins(%[[S21]], %[[S19]] : tensor<4x4xf32>, tensor<4x4xf32>) outs(%[[S20]] : tensor<4x4xf32>) -> tensor<4x4xf32>
+// CHECK:          %[[INSERTED_SLICE_9:.*]] = tensor.insert_slice %[[S22]] into %[[ARG10]][%[[ARG7]], 0, 0, %[[ARG9]]] [1, 4, 4, 1] [1, 1, 1, 1]
+// CHECK:          scf.yield %[[INSERTED_SLICE_9]]
+// CHECK:        scf.yield %[[S15]]
+// CHECK:      %[[S13:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
+// CHECK:      %[[S14:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG6]][0, %[[S13]], %[[S14]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+
+// -----
+
+func.func @conv2d_unaligned(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<2x9x9x2xf32>) -> tensor<2x9x9x2xf32> {
+  %cst = arith.constant 0.000000e+00 : f32
+  %0 = tensor.empty() : tensor<6x6x5x2xf32>
+  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %padded = tensor.pad %arg0 low[0, 0, 0, 0] high[0, 3, 3, 0] {
+  ^bb0(%arg4: index, %arg5: index, %arg6: index, %arg7: index):
+    tensor.yield %cst : f32
+  } : tensor<2x11x11x5xf32> to tensor<2x14x14x5xf32>
+  %2 = tensor.empty() : tensor<6x6x3x3x2x5xf32>
+  %3 = linalg.winograd_input_transform m(4) r(3) ins(%padded : tensor<2x14x14x5xf32>) outs(%2 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
+  %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
+  %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x6x3x3x2x5xf32> into tensor<36x18x5xf32>
+  %4 = tensor.empty() : tensor<36x18x2xf32>
+  %5 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<36x18x5xf32>, tensor<36x5x2xf32>) outs(%4 : tensor<36x18x2xf32>) -> tensor<36x18x2xf32>
+  %expanded = tensor.expand_shape %5 [[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 3, 3, 2, 2] : tensor<36x18x2xf32> into tensor<6x6x3x3x2x2xf32>
+  %padded_1 = tensor.pad %arg2 low[0, 0, 0, 0] high[0, 3, 3, 0] {
+  ^bb0(%arg4: index, %arg5: index, %arg6: index, %arg7: index):
+    tensor.yield %cst : f32
+  } : tensor<2x9x9x2xf32> to tensor<2x12x12x2xf32>
+  %6 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x6x3x3x2x2xf32>) outs(%padded_1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
+  %extracted_slice = tensor.extract_slice %6[0, 0, 0, 0] [2, 9, 9, 2] [1, 1, 1, 1] : tensor<2x12x12x2xf32> to tensor<2x9x9x2xf32>
+  return %extracted_slice : tensor<2x9x9x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %3, %loop3:2 = transform.structured.tile_using_for %2 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %4 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %5, %loop5:2 = transform.structured.tile_using_for %4 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %7 = transform.structured.decompose_winograd_op %0 : (!transform.any_op) -> (!transform.any_op)
+    %8 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %3 : (!transform.any_op) -> !transform.any_op
+    %9 = transform.structured.decompose_winograd_op %8 : (!transform.any_op) -> (!transform.any_op)
+    %10 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %5 : (!transform.any_op) -> !transform.any_op
+    %11 = transform.structured.decompose_winograd_op %10 : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1) -> ()>
+// CHECK: #[[$MAP2:.+]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK-LABEL: func.func @conv2d_unaligned
+// CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x11x11x5xf32>, %[[ARG1:.*]]: tensor<2x3x3x5xf32>, %[[ARG2:.*]]: tensor<2x9x9x2xf32>) -> tensor<2x9x9x2xf32> {
+// CHECK:  %[[CST:.*]] = arith.constant 1.024000e+03 : f32
+// CHECK:  %[[C3:.*]] = arith.constant 3 : index
+// CHECK:  %[[C1:.*]] = arith.constant 1 : index
+// CHECK:  %[[C5:.*]] = arith.constant 5 : index
+// CHECK:  %[[C2:.*]] = arith.constant 2 : index
+// CHECK:  %[[C0:.*]] = arith.constant 0 : index
+// CHECK:  %[[S0:.*]] = tensor.empty()
+// CHECK:  %[[S1:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S0]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG6:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:      %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG4]], 0, 0, %[[ARG6]]] [1, 3, 3, 1] [1, 1, 1, 1]
+// CHECK:      %[[S11:.*]] = linalg.matmul
+// CHECK:      %[[S13:.*]] = linalg.matmul
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S13]] into %[[ARG7]][0, 0, %[[ARG6]], %[[ARG4]]] [6, 6, 1, 1] [1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]] : tensor<6x6x5x2xf32>
+// CHECK:    scf.yield %[[S9]] : tensor<6x6x5x2xf32>
+// CHECK:  %[[PADDED:.*]] = tensor.pad %[[ARG0]] low[0, 0, 0, 0] high[0, 3, 3, 0]
+// CHECK:  %[[S2:.*]] = tensor.empty() : tensor<6x6x3x3x2x5xf32>
+// CHECK:  %[[S3:.*]] = tensor.empty() : tensor<6x6x3x3x2x5xf32>
+// CHECK:  %[[S4:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S3]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG6:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG6]])
+// CHECK:      %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[PADDED]][0, %[[S10]], %[[S11]], 0] [2, 6, 6, 5] [1, 1, 1, 1]
+// CHECK:      %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[S2]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[S12:.*]] = scf.for %[[ARG8:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG9:.*]] = %[[EXTRACTED_SLICE_10]])
+// CHECK:        %[[S13:.*]] = scf.for %[[ARG10:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG11:.*]] = %[[ARG9]])
+// CHECK:          %[[EXTRACTED_SLICE_11:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE_9]][%[[ARG8]], 0, 0, %[[ARG10]]] [1, 6, 6, 1] [1, 1, 1, 1]
+// CHECK:          %[[S15:.*]] = linalg.matmul
+// CHECK:          %[[S17:.*]] = linalg.matmul
+// CHECK:          %[[INSERTED_SLICE_12:.*]] = tensor.insert_slice %[[S17]] into %[[ARG11]][0, 0, 0, 0, %[[ARG8]], %[[ARG10]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          scf.yield %[[INSERTED_SLICE_12]] : tensor<6x6x1x1x2x5xf32>
+// CHECK:        scf.yield %[[S13]] : tensor<6x6x1x1x2x5xf32>
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG7]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+// CHECK:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
+// CHECK:  %[[COLLAPSED_7:.*]] = tensor.collapse_shape %[[S4]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:  %[[S6:.*]] = linalg.batch_matmul
+// CHECK:  %[[EXPANDED:.*]] = tensor.expand_shape %[[S6]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 3, 3, 2, 2]
+// CHECK:  %[[PADDED_8:.*]] = tensor.pad %[[ARG2]] low[0, 0, 0, 0] high[0, 3, 3, 0]
+// CHECK:  %[[S7:.*]] = tensor.empty() : tensor<2x12x12x2xf32>
+// CHECK:  %[[S8:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S7]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG6:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:      %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG6]])
+// CHECK:      %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[PADDED_8]][0, %[[S10]], %[[S11]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      %[[S12:.*]] = scf.for %[[ARG8:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG9:.*]] = %[[EXTRACTED_SLICE_10]])
+// CHECK:        %[[S15:.*]] = scf.for %[[ARG10:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG11:.*]] = %[[ARG9]])
+// CHECK:          %[[EXTRACTED_SLICE_11:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE_9]][0, 0, 0, 0, %[[ARG8]], %[[ARG10]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          %[[S17:.*]] = linalg.matmul
+// CHECK:          %[[S19:.*]] = linalg.matmul
+// CHECK:          %[[S20:.*]] = tensor.empty() : tensor<4x4xf32>
+// CHECK:          %[[S21:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S20]] : tensor<4x4xf32>) {
+// CHECK:          ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:            linalg.yield %[[IN]] : f32
+// CHECK:          } -> tensor<4x4xf32>
+// CHECK:          %[[S22:.*]] = linalg.mul ins(%[[S21]], %[[S19]] : tensor<4x4xf32>, tensor<4x4xf32>) outs(%[[S20]] : tensor<4x4xf32>) -> tensor<4x4xf32>
+// CHECK:          %[[INSERTED_SLICE_12:.*]] = tensor.insert_slice %[[S22]] into %[[ARG11]][%[[ARG8]], 0, 0, %[[ARG10]]] [1, 4, 4, 1] [1, 1, 1, 1]
+// CHECK:          scf.yield %[[INSERTED_SLICE_12]]
+// CHECK:        scf.yield %[[S15]] : tensor<2x4x4x2xf32>
+// CHECK:      %[[S13:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:      %[[S14:.*]] = affine.apply #[[$MAP0]](%[[ARG6]])
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG7]][0, %[[S13]], %[[S14]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+// CHECK:  %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[S8]][0, 0, 0, 0] [2, 9, 9, 2] [1, 1, 1, 1]
+// CHECK:  return %[[EXTRACTED_SLICE]]
+
+// -----
+
+func.func @conv2d_mx1_rx1(%arg0: tensor<2x6x1x5xf32>, %arg1: tensor<2x3x1x5xf32>, %arg2: tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32> {
+  %0 = tensor.empty() : tensor<6x1x5x2xf32>
+  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x1x5xf32>) outs(%0 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+  %2 = tensor.empty() : tensor<6x1x1x1x2x5xf32>
+  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x6x1x5xf32>) outs(%2 : tensor<6x1x1x1x2x5xf32>) -> tensor<6x1x1x1x2x5xf32>
+  %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x1x5x2xf32> into tensor<6x5x2xf32>
+  %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x1x1x1x2x5xf32> into tensor<6x2x5xf32>
+  %4 = tensor.empty() : tensor<6x2x2xf32>
+  %5 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<6x2x5xf32>, tensor<6x5x2xf32>) outs(%4 : tensor<6x2x2xf32>) -> tensor<6x2x2xf32>
+  %expanded = tensor.expand_shape %5 [[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 1, 2, 2] : tensor<6x2x2xf32> into tensor<6x1x1x1x2x2xf32>
+  %6 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x1x1x1x2x2xf32>) outs(%arg2 : tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32>
+  return %6 : tensor<2x4x1x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %3, %loop3:2 = transform.structured.tile_using_for %2 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %4 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %5, %loop5:2 = transform.structured.tile_using_for %4 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %7 = transform.structured.decompose_winograd_op %0 : (!transform.any_op) -> (!transform.any_op)
+    %8 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %3 : (!transform.any_op) -> !transform.any_op
+    %9 = transform.structured.decompose_winograd_op %8 : (!transform.any_op) -> (!transform.any_op)
+    %10 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %5 : (!transform.any_op) -> !transform.any_op
+    %11 = transform.structured.decompose_winograd_op %10 : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0, d1) -> ()>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK-LABEL: func.func @conv2d_mx1_rx1
+// CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x6x1x5xf32>, %[[ARG1:.*]]: tensor<2x3x1x5xf32>, %[[ARG2:.*]]: tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32> {
+// CHECK:   %[[CST:.*]] = arith.constant 3.200000e+01 : f32
+// CHECK:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK:   %[[S0:.*]] = tensor.empty() : tensor<6x1x5x2xf32>
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S0]])
+// CHECK:     %[[S7:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 3, 1, 1] [1, 1, 1, 1]
+// CHECK:       %[[S9:.*]] = linalg.matmul
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG6]][0, 0, %[[ARG5]], %[[ARG3]]] [6, 1, 1, 1] [1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE]]
+// CHECK:     scf.yield %[[S7]]
+// CHECK:   %[[S2:.*]] = tensor.empty() : tensor<6x1x1x1x2x5xf32>
+// CHECK:   %[[S3:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S2]])
+// CHECK:     %[[S7:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 6, 1, 1] [1, 1, 1, 1]
+// CHECK:       %[[S9:.*]] = linalg.matmul
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG6]][0, 0, 0, 0, %[[ARG3]], %[[ARG5]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE]]
+// CHECK:     scf.yield %[[S7]]
+// CHECK:   %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
+// CHECK:   %[[COLLAPSED_3:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:   %[[S5:.*]] = linalg.batch_matmul
+// CHECK:   %[[EXPANDED:.*]] = tensor.expand_shape %[[S5]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 1, 2, 2]
+// CHECK:   %[[S6:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[ARG2]])
+// CHECK:     %[[S7:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, 0, 0, %[[ARG3]], %[[ARG5]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:       %[[S9:.*]] = linalg.matmul
+// CHECK:       %[[S10:.*]] = tensor.empty() : tensor<4x1xf32>
+// CHECK:       %[[S11:.*]] = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S10]] : tensor<4x1xf32>) {
+// CHECK:       ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:         linalg.yield %[[IN]] : f32
+// CHECK:       } -> tensor<4x1xf32>
+// CHECK:       %[[S12:.*]] = linalg.mul ins(%[[S11]], %[[S9]] : tensor<4x1xf32>, tensor<4x1xf32>) outs(%[[S10]] : tensor<4x1xf32>) -> tensor<4x1xf32>
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG6]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 4, 1, 1] [1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE]]
+// CHECK:     scf.yield %[[S7]]
+// CHECK:   return %[[S6]]
diff --git a/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
new file mode 100644
index 000000000000000..21522a2083b463e
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
@@ -0,0 +1,380 @@
+// RUN: mlir-opt %s -transform-interpreter --split-input-file | FileCheck %s
+
+func.func @tile_winograd_filter(%arg0: tensor<2x3x3x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
+  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  return %0 : tensor<6x6x5x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:2 = transform.structured.tile_using_for %0 tile_sizes [1, 0, 0, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK-LABEL: func.func @tile_winograd_filter(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x3x3x5xf32>, %[[ARG1:.*]]: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
+// CHECK-DAG:  %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:  %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:  %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:  %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:  %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:  %[[C1_1:.*]] = arith.constant 1 : index
+// CHECK:  %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:    %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]]
+// CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 3, 1] [1, 1, 1, 1] : tensor<2x3x3x5xf32> to tensor<1x3x3x1xf32>
+// CHECK:      %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, 1, 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x1x1xf32>
+// CHECK:      %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x6x1x1xf32>) -> tensor<6x6x1x1xf32>
+
+// -----
+
+func.func @tile_winograd_filter(%arg0: tensor<2x3x3x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
+  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  return %0 : tensor<6x6x5x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:2 = transform.structured.tile_using_for %0 tile_sizes [1, 0, 0, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (-d0 + 5, 2)>
+// CHECK-LABEL: func.func @tile_winograd_filter(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x3x3x5xf32>, %[[ARG1:.*]]: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C2_1:.*]] = arith.constant 2 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C2_1]]
+// CHECK:       %[[C5_2:.*]] = arith.constant 5 : index
+// CHECK:       %[[S3:.*]] = affine.min #[[$MAP0]](%[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 3, %[[S3]]] [1, 1, 1, 1] : tensor<2x3x3x5xf32> to tensor<1x3x3x?xf32>
+// CHECK:       %[[EXTRACTED_SLICE_3:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, %[[S3]], 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x?x1xf32>
+// CHECK:       %[[S4:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x?xf32>) outs(%[[EXTRACTED_SLICE_3]] : tensor<6x6x?x1xf32>) -> tensor<6x6x?x1xf32>
+
+// -----
+
+func.func @tile_winograd_filter(%arg0: tensor<2x3x1x5xf32>, %arg1: tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32> {
+  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x1x5xf32>) outs(%arg1 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+  return %0 : tensor<6x1x5x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:2 = transform.structured.tile_using_for %0 tile_sizes [1, 0, 0, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK-LABEL: func.func @tile_winograd_filter(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x3x1x5xf32>, %[[ARG1:.*]]: tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_1:.*]] = arith.constant 1 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]]
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 1, 1] [1, 1, 1, 1] : tensor<2x3x1x5xf32> to tensor<1x3x1x1xf32>
+// CHECK:       %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 1, 1, 1] [1, 1, 1, 1] : tensor<6x1x5x2xf32> to tensor<6x1x1x1xf32>
+// CHECK:       %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x1x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x1x1x1xf32>) -> tensor<6x1x1x1xf32>
+
+// -----
+
+func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  return %0 : tensor<6x6x2x2x2x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop3:2 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (6)>
+// CHECK-LABEL: func.func @tile_winograd_input(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_1:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK: %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
+// CHECK:   %[[S3:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:   %[[S4:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:   %[[S5:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:   %[[S6:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:   %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, %[[S3]], %[[S4]], 0] [2, %[[S5]], %[[S6]], 5] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<2x?x?x5xf32>
+// CHECK:   %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x2x5xf32>
+// CHECK:   %[[S7:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x5xf32>) outs(%[[EXTRACTED_SLICE_5]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
+
+// -----
+
+func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  return %0 : tensor<6x6x2x2x2x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop3:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (6)>
+// CHECK-LABEL: func.func @tile_winograd_input(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_3:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_6:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_1:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_4:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[C1_5:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[C1_7:.*]] = arith.constant 1 : index
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:   %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
+// CHECK:     %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C2_4]] step %[[C1_5]]
+// CHECK:       %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C1_7]]
+// CHECK:         %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:         %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:         %[[S7:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:         %[[S8:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S5]], %[[S6]], %[[ARG8]]] [1, %[[S7]], %[[S8]], 1] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<1x?x?x1xf32>
+// CHECK:         %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x1x1xf32>
+// CHECK:         %[[S9:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x?x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<6x6x1x1x1x1xf32>) -> tensor<6x6x1x1x1x1xf32>
+
+// -----
+
+func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  return %0 : tensor<6x6x2x2x2x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop3:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 2, 2, 2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (-d0 + 5, 2)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP2:.+]] = affine_map<() -> (10)>
+// CHECK-LABEL: func.func @tile_winograd_input(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_4:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_7:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_5:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG: %[[C2_0:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_3:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_6:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_8:.*]] = arith.constant 2 : index
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C2_0]]
+// CHECK:   %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2_2]] step %[[C2_3]]
+// CHECK:     %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_4]] to %[[C2_5]] step %[[C2_6]]
+// CHECK:       %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_7]] to %[[C5]] step %[[C2_8]]
+// CHECK:         %[[S5:.*]] = affine.min #[[$MAP0]](%[[ARG8]])
+// CHECK:         %[[S6:.*]] = affine.apply #[[$MAP1]](%[[ARG2]])
+// CHECK:         %[[S7:.*]] = affine.apply #[[$MAP1]](%[[ARG4]])
+// CHECK:         %[[S8:.*]] = affine.apply #[[$MAP2]]()
+// CHECK:         %[[S9:.*]] = affine.apply #[[$MAP2]]()
+// CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S6]], %[[S7]], %[[ARG8]]] [2, %[[S8]], %[[S9]], %[[S5]]] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<2x?x?x?xf32>
+// CHECK:         %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, 2, %[[S5]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x2x2x2x?xf32>
+// CHECK:         %[[S10:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x?xf32>) outs(%[[EXTRACTED_SLICE_12]] : tensor<6x6x2x2x2x?xf32>) -> tensor<6x6x2x2x2x?xf32>
+
+// -----
+
+func.func @tile_winograd_input(%arg0: tensor<2x1x10x5xf32>, %arg1: tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32> {
+  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x1x10x5xf32>) outs(%arg1 : tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32>
+  return %0 : tensor<1x6x1x2x2x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop3:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (6)>
+// CHECK-LABEL: func.func @tile_winograd_input(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x1x10x5xf32>, %[[ARG1:.*]]: tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_3:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_6:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C2_4:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:   %[[C1_0:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_5:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_7:.*]] = arith.constant 1 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C1]] step %[[C1_0]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2]] step %[[C1_2]]
+// CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C2_4]] step %[[C1_5]]
+// CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C1_7]]
+// CHECK:           %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:           %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:           %[[S7:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:           %[[S8:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], 0, %[[S6]], %[[ARG8]]] [1, 1, %[[S8]], 1] [1, 1, 1, 1] : tensor<2x1x10x5xf32> to tensor<1x1x?x1xf32>
+// CHECK:           %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [1, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<1x6x1x2x2x5xf32> to tensor<1x6x1x1x1x1xf32>
+// CHECK:           %[[S9:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x1x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<1x6x1x1x1x1xf32>) -> tensor<1x6x1x1x1x1xf32>
+
+// -----
+
+func.func @tile_winograd_output(%arg0 : tensor<6x6x2x2x2x2xf32>, %arg1: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
+  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x2x2x2x2xf32>) outs(%arg1 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
+  return %0 : tensor<2x8x8x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:2 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (4)>
+// CHECK-LABEL: func.func @tile_winograd_output(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<6x6x2x2x2x2xf32>, %[[ARG1:.*]]: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C2_1:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x2xf32> to tensor<6x6x1x1x2x2xf32>
+// CHECK:       %[[S3:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:       %[[S4:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:       %[[S5:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:       %[[S6:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:       %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG1]][0, %[[S3]], %[[S4]], 0] [2, %[[S5]], %[[S6]], 2] [1, 1, 1, 1] : tensor<2x8x8x2xf32> to tensor<2x?x?x2xf32>
+
+// -----
+
+func.func @tile_winograd_output(%arg0 : tensor<6x6x2x2x3x5xf32>, %arg1: tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32> {
+  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x2x2x3x5xf32>) outs(%arg1 : tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32>
+  return %0 : tensor<3x8x8x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 2, 2, 2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (-d0 + 3, 2)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0) -> (-d0 + 5, 2)>
+// CHECK: #[[$MAP2:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP3:.+]] = affine_map<() -> (8)>
+// CHECK-LABEL:  func.func @tile_winograd_output(
+// CHECK-SAME:   %[[ARG0:.*]]: tensor<6x6x2x2x3x5xf32>, %[[ARG1:.*]]: tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32> {
+// CHECK-DAG:    %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:    %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG:    %[[C0_4:.*]] = arith.constant 0 : index
+// CHECK-DAG:    %[[C0_6:.*]] = arith.constant 0 : index
+// CHECK-DAG:    %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C2_2:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C3:.*]] = arith.constant 3 : index
+// CHECK-DAG:    %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:    %[[C2_0:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C2_3:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C2_5:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C2_7:.*]] = arith.constant 2 : index
+// CHECK:    %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C2_0]]
+// CHECK:      %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2_2]] step %[[C2_3]]
+// CHECK:        %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_4]] to %[[C3]] step %[[C2_5]]
+// CHECK:          %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C2_7]]
+// CHECK:            %[[C3_8:.*]] = arith.constant 3 : index
+// CHECK:            %[[S5:.*]] = affine.min #[[$MAP0]](%[[ARG6]])
+// CHECK:            %[[C5_9:.*]] = arith.constant 5 : index
+// CHECK:            %[[S6:.*]] = affine.min #[[$MAP1]](%[[ARG8]])
+// CHECK:            %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, %[[S5]], %[[S6]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x3x5xf32> to tensor<6x6x2x2x?x?xf32>
+// CHECK:            %[[S7:.*]] = affine.apply #[[$MAP2]](%[[ARG2]])
+// CHECK:            %[[S8:.*]] = affine.apply #[[$MAP2]](%[[ARG4]])
+// CHECK:            %[[S9:.*]] = affine.apply #[[$MAP3]]()
+// CHECK:            %[[S10:.*]] = affine.apply #[[$MAP3]]()
+// CHECK:            %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG6]], %[[S7]], %[[S8]], %[[ARG8]]] [%[[S5]], %[[S9]], %[[S10]], %[[S6]]] [1, 1, 1, 1] : tensor<3x8x8x5xf32> to tensor<?x?x?x?xf32>
+
+// -----
+
+func.func @tile_winograd_output(%arg0 : tensor<6x1x2x1x3x5xf32>, %arg1: tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32> {
+  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x1x2x1x3x5xf32>) outs(%arg1 : tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32>
+  return %0 : tensor<3x8x1x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (4)>
+// CHECK-LABEL: func.func @tile_winograd_output(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<6x1x2x1x3x5xf32>, %[[ARG1:.*]]: tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_3:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_5:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C1_1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C3:.*]] = arith.constant 3 : index
+// CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_4:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_6:.*]] = arith.constant 1 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C1_1]] step %[[C1_2]]
+// CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C3]] step %[[C1_4]]
+// CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_5]] to %[[C5]] step %[[C1_6]]
+// CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x1x2x1x3x5xf32> to tensor<6x1x1x1x1x1xf32>
+// CHECK:           %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:           %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:           %[[S7:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:           %[[S8:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:           %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG6]], %[[S5]], 0, %[[ARG8]]] [1, %[[S7]], 1, 1] [1, 1, 1, 1] : tensor<3x8x1x5xf32> to tensor<1x?x1x1xf32>
+// CHECK:           %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<6x1x1x1x1x1xf32>) outs(%[[EXTRACTED_SLICE_9]] : tensor<1x?x1x1xf32>) -> tensor<1x?x1x1xf32>
diff --git a/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir b/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
index f7fd05b36bae062..d8a26c71d12f91d 100644
--- a/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
@@ -28,6 +28,62 @@ func.func @group_non_uniform_ballot(%predicate: i1) -> vector<4xsi32> {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotFindLSB
+//===----------------------------------------------------------------------===//
+
+func.func @group_non_uniform_ballot_find_lsb(%value : vector<4xi32>) -> i32 {
+  // CHECK: %{{.*}} = spirv.GroupNonUniformBallotFindLSB <Subgroup> %{{.*}}: vector<4xi32>, i32
+  %0 = spirv.GroupNonUniformBallotFindLSB <Subgroup> %value : vector<4xi32>, i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_find_lsb(%value : vector<4xi32>) -> i32 {
+  // expected-error @+1 {{execution scope must be 'Workgroup' or 'Subgroup'}}
+  %0 = spirv.GroupNonUniformBallotFindLSB <Device> %value : vector<4xi32>, i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_find_lsb(%value : vector<4xi32>) -> si32 {
+  // expected-error @+1 {{op result #0 must be 8/16/32/64-bit signless/unsigned integer, but got 'si32'}}
+  %0 = spirv.GroupNonUniformBallotFindLSB <Subgroup> %value : vector<4xi32>, si32
+  return %0: si32
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotFindLSB
+//===----------------------------------------------------------------------===//
+
+func.func @group_non_uniform_ballot_find_msb(%value : vector<4xi32>) -> i32 {
+  // CHECK: %{{.*}} = spirv.GroupNonUniformBallotFindMSB <Subgroup> %{{.*}}: vector<4xi32>, i32
+  %0 = spirv.GroupNonUniformBallotFindMSB <Subgroup> %value : vector<4xi32>, i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_find_msb(%value : vector<4xi32>) -> i32 {
+  // expected-error @+1 {{execution scope must be 'Workgroup' or 'Subgroup'}}
+  %0 = spirv.GroupNonUniformBallotFindMSB <Device> %value : vector<4xi32>, i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_find_msb(%value : vector<4xi32>) -> si32 {
+  // expected-error @+1 {{op result #0 must be 8/16/32/64-bit signless/unsigned integer, but got 'si32'}}
+  %0 = spirv.GroupNonUniformBallotFindMSB <Subgroup> %value : vector<4xi32>, si32
+  return %0: si32
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.NonUniformGroupBroadcast
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index 3ac473d3a9d8c06..54d299a3efcecd7 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -405,6 +405,14 @@ func.func @test_reshape_invalid_placeholders(%arg0 : tensor<?xf32>) -> () {
 
 // -----
 
+func.func @test_reshape_invalid_tensor_dim(%arg0 : tensor<4x?xf32>) -> () {
+  // expected-error@+1 {{'tosa.reshape' op new shape has invalid tensor dimension size -2}}
+  %0 = "tosa.reshape" (%arg0) {new_shape = array<i64: -2, -1>} : (tensor<4x?xf32>) -> tensor<?x4xf32>
+  return
+}
+
+// -----
+
 func.func @test_reverse_axis_out_of_range(%arg0 : tensor<13x21x3xf32>) -> () {
   // expected-error@+1 {{'tosa.reverse' op expect input tensor rank (3) to be larger than reverse axis (5)}}
   %0 = tosa.reverse %arg0 {axis = 5 : i32} : (tensor<13x21x3xf32>) -> tensor<?x?x?xi32>
diff --git a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
index bcc8bd1c039e430..d92be038071b08b 100644
--- a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
@@ -1373,4 +1373,58 @@ func.func @test_tosa_use_def_chain(%arg0: tensor<1x32x32x3xf32>, %arg1: tensor<1
   // CHECK: (tensor<1x32x32x16xf32>) -> tensor<1x16x16x16xf32>
   %1 = tosa.max_pool2d %0 {kernel = array<i64: 2, 2>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 2, 2>} : (tensor<?x32x32x16xf32>) -> tensor<?x16x16x16xf32>
   return %1 : tensor<?x16x16x16xf32>
-}
\ No newline at end of file
+}
+
+// -----
+
+// This test locks two bug fixes manifested in the code below.
+//
+// 1. Context
+//
+// When shape propagation hits an operation that does not support shape
+// inference (here 'tensor.expand_shape'), it must revert the currently
+// inferred shape of its consumers back to the originally expected input
+// type to avoid potential op verification errors. This type reversal is
+// done through an additional 'tensor.cast' op.
+//
+//
+// 2. Preserving list of non-inferrable consumers
+//
+// When multiple non-inferrable consumers of a shape-inferred value are found
+// (here, the 2 occurrences of 'tensor.expand_shape' consuming the output of
+// 'tosa.cast'), their input argument ('%0') must be altered to consume the
+// output the new 'tensor.cast' op. While these replacements occur, the use list
+// of the producer ('tosa.cast') is also implicitly altered, invalidating any
+// iterators associated with it. It is therefore necessary to create a copy of
+// this use list ahead of time. Before this bug fix, the second
+// 'tensor.expand_shape' op below was not updated correctly.
+//
+// 3. Guaranteeing def-use order
+//
+// When emitting the 'tensor.cast' op, it is important to guarantee that its
+// output value is defined before all of its consumers (here, both of the
+// 'tensor.expand_shape' ops. In a previous version of the code, this insertion
+// occurred right before the first encountered consumer. Since use lists are
+// saved in reverse order, the 'tensor.cast' op was inserted before the second
+// 'tensor.expand_shape' op, leading to a def-use order violation when the
+// first 'tensor.expand_shape' op was later updated. The current implementation
+// sets the insertion point right after the producer of the last shape-inferred
+// value (here 'tosa.cast'), which guarantees correct def-use order for all
+// future operand updates.
+
+// CHECK-LABEL: test_multiple_non_inferrable_consumers
+// CHECK-SAME: %[[ARG:.*]]: tensor<1x2x8xf32>
+func.func @test_multiple_non_inferrable_consumers(%arg0: tensor<1x2x8xf32>) {
+  // CHECK: %[[TOSA_CAST:.*]] = tosa.cast %[[ARG]] : (tensor<1x2x8xf32>) -> tensor<1x2x8xf32>
+  // CHECK: %[[TENSOR_CAST:.*]] = tensor.cast %[[TOSA_CAST]] : tensor<1x2x8xf32> to tensor<?x2x8xf32>
+  %0 = tosa.cast %arg0 : (tensor<1x2x8xf32>) -> tensor<?x2x8xf32>
+
+  %c0 = arith.constant 0 : index
+  %dim = tensor.dim %0, %c0 : tensor<?x2x8xf32>
+
+  // CHECK: tensor.expand_shape %[[TENSOR_CAST]]
+  // CHECK: tensor.expand_shape %[[TENSOR_CAST]]
+  %expanded_0 = tensor.expand_shape %0 [[0], [1, 2], [3]] output_shape [%dim, 1, 4, 8] : tensor<?x2x8xf32> into tensor<?x1x2x8xf32>
+  %expanded_1 = tensor.expand_shape %0 [[0], [1, 2], [3]] output_shape [%dim, 1, 4, 8] : tensor<?x2x8xf32> into tensor<?x1x2x8xf32>
+  return
+}
diff --git a/mlir/test/Dialect/Vector/vector-reduce-to-contract.mlir b/mlir/test/Dialect/Vector/vector-reduce-to-contract.mlir
index c0dbea81df892a8..24070dbf017a58f 100644
--- a/mlir/test/Dialect/Vector/vector-reduce-to-contract.mlir
+++ b/mlir/test/Dialect/Vector/vector-reduce-to-contract.mlir
@@ -245,128 +245,6 @@ func.func @contract_broadcast_would_have_no_reduction_dim_pair(%arg0 : vector<1x
 }
 
 
-//===----------------------------------------------------------------------===//
-// [Pattern: ReorderCastOpsOnBroadcast]
-//
-// Reorder casting ops and vector ops. The casting ops have almost identical
-// pattern, so only arith.extsi op is tested.
-//
-// TODO: Potential duplication with sink-vector-broadcast.mlir
-//===----------------------------------------------------------------------===//
-
-// -----
-
-func.func @broadcast_vector_extsi(%a : vector<4xi8>) -> vector<2x4xi32> {
-  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : vector<4xi8> to vector<4xi32>
-  // CHECK: vector.broadcast %[[EXT:.+]] : vector<4xi32> to vector<2x4xi32>
-  %b = vector.broadcast %a : vector<4xi8> to vector<2x4xi8>
-  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
-  return %r : vector<2x4xi32>
-}
-
-// -----
-
-func.func @broadcast_scalar_extsi(%a : i8) -> vector<2x4xi32> {
-  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : i8 to i32
-  // CHECK: vector.broadcast %[[EXT]] : i32 to vector<2x4xi32>
-  %b = vector.broadcast %a : i8 to vector<2x4xi8>
-  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
-  return %r : vector<2x4xi32>
-}
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// [Pattern: ReorderElementwiseOpsOnTranspose]
-//
-// TODO: Potential duplication with sink-vector-broadcast.mlir
-//===----------------------------------------------------------------------===//
-func.func @transpose_extsi(%a : vector<4x2xi8>) -> vector<2x4xi32> {
-  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : vector<4x2xi8> to vector<4x2xi32>
-  // CHECK: vector.transpose %[[EXT]], [1, 0] : vector<4x2xi32> to vector<2x4xi32>
-  %b = vector.transpose %a, [1, 0]: vector<4x2xi8> to vector<2x4xi8>
-  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
-  return %r : vector<2x4xi32>
-}
-
-//===----------------------------------------------------------------------===//
-// Reorder elementwise ops and vector ops.
-// TODO: Potential duplication with sink-vector-broadcast.mlir
-//===----------------------------------------------------------------------===//
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_same_type
-//  CHECK-SAME: (%[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
-//       CHECK:   %[[ADD:.+]] = arith.addf %[[A]], %[[B]] : vector<4x2xf32>
-//       CHECK:   %[[T:.+]] = vector.transpose %[[ADD]], [1, 0]
-//       CHECK:   return %[[T]]
-
-func.func @transpose_elementwise_same_type(%a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xf32> {
-  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %r = arith.addf %at, %bt : vector<2x4xf32>
-  return %r : vector<2x4xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_diff_operand_types
-//  CHECK-SAME: (%[[COND:.+]]: vector<4x2xi1>, %[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
-//       CHECK:   %[[S:.+]] = arith.select %[[COND]], %[[A]], %[[B]] : vector<4x2xi1>, vector<4x2xf32>
-//       CHECK:   %[[T:.+]] = vector.transpose %[[S]], [1, 0] : vector<4x2xf32> to vector<2x4xf32>
-//       CHECK:   return %[[T]]
-func.func @transpose_elementwise_diff_operand_types(%cond: vector<4x2xi1>, %a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xf32> {
-  %condt = vector.transpose %cond, [1, 0]: vector<4x2xi1> to vector<2x4xi1>
-  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %r = arith.select %condt, %at, %bt : vector<2x4xi1>, vector<2x4xf32>
-  return %r : vector<2x4xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_diff_operand_result_type
-//  CHECK-SAME: (%[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
-//       CHECK:   %[[CMP:.+]] = arith.cmpf olt, %[[A]], %[[B]] : vector<4x2xf32>
-//       CHECK:   %[[T:.+]] = vector.transpose %[[CMP]], [1, 0] : vector<4x2xi1> to vector<2x4xi1>
-//       CHECK:   return %[[T]]
-func.func @transpose_elementwise_diff_operand_result_type(%a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xi1> {
-  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %r = arith.cmpf olt, %at, %bt : vector<2x4xf32>
-  return %r : vector<2x4xi1>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_splat_constant
-//  CHECK-SAME: (%[[A:.+]]: vector<4x6x3x2xf32>)
-//       CHECK:   %[[B:.+]] = arith.constant dense<5.000000e+00> : vector<4x6x3x2xf32>
-//       CHECK:   %[[ADD:.+]] = arith.addf %[[A]], %[[B]] : vector<4x6x3x2xf32>
-//       CHECK:   %[[T:.+]] = vector.transpose %[[ADD]], [1, 0, 3, 2] : vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
-//       CHECK:   return %[[T:.+]] : vector<6x4x2x3xf32>
-
-func.func @transpose_elementwise_splat_constant(%a : vector<4x6x3x2xf32>) -> vector<6x4x2x3xf32> {
-  %b = arith.constant dense<5.0> : vector<6x4x2x3xf32>
-  %at = vector.transpose %a, [1, 0, 3, 2]: vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
-  %r = arith.addf %at, %b : vector<6x4x2x3xf32>
-  return %r : vector<6x4x2x3xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_diff_map
-//       CHECK:   vector.transpose
-//       CHECK:   vector.transpose
-//       CHECK:   arith.addf
-func.func @transpose_elementwise_diff_map(%a : vector<4x6x3x2xf32>, %b: vector<6x2x4x3xf32>) -> vector<6x4x2x3xf32> {
-  %at = vector.transpose %a, [1, 0, 3, 2]: vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
-  %bt = vector.transpose %b, [0, 2, 1, 3]: vector<6x2x4x3xf32> to vector<6x4x2x3xf32>
-  %r = arith.addf %at, %bt : vector<6x4x2x3xf32>
-  return %r : vector<6x4x2x3xf32>
-}
-
 // -----
 
 // CHECK-DAG: #[[$LHS_MAP:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>
diff --git a/mlir/test/Dialect/Vector/sink-vector-broadcast.mlir b/mlir/test/Dialect/Vector/vector-sink.mlir
similarity index 65%
rename from mlir/test/Dialect/Vector/sink-vector-broadcast.mlir
rename to mlir/test/Dialect/Vector/vector-sink.mlir
index dd2e98831a708a0..5a3699333265c1c 100644
--- a/mlir/test/Dialect/Vector/sink-vector-broadcast.mlir
+++ b/mlir/test/Dialect/Vector/vector-sink.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-sink-vector-broadcast -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -test-vector-sink-patterns -split-input-file | FileCheck %s
 
 //-----------------------------------------------------------------------------
 // [Pattern: ReorderElementwiseOpsOnBroadcast]
@@ -208,3 +208,115 @@ func.func @negative_op_only_supports_vectors(%arg0 : f32) -> vector<1xf32> {
   %1 = vector.fma %0, %0, %0 : vector<1xf32>
   return %1 : vector<1xf32>
 }
+
+//===----------------------------------------------------------------------===//
+// [Pattern: ReorderCastOpsOnBroadcast]
+//
+// Reorder casting ops and vector ops. The casting ops have almost identical
+// pattern, so only arith.extsi op is tested.
+//===----------------------------------------------------------------------===//
+
+// -----
+
+func.func @broadcast_vector_extsi(%a : vector<4xi8>) -> vector<2x4xi32> {
+  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : vector<4xi8> to vector<4xi32>
+  // CHECK: vector.broadcast %[[EXT:.+]] : vector<4xi32> to vector<2x4xi32>
+  %b = vector.broadcast %a : vector<4xi8> to vector<2x4xi8>
+  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
+  return %r : vector<2x4xi32>
+}
+
+// -----
+
+func.func @broadcast_scalar_extsi(%a : i8) -> vector<2x4xi32> {
+  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : i8 to i32
+  // CHECK: vector.broadcast %[[EXT]] : i32 to vector<2x4xi32>
+  %b = vector.broadcast %a : i8 to vector<2x4xi8>
+  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
+  return %r : vector<2x4xi32>
+}
+
+//===----------------------------------------------------------------------===//
+// [Pattern: ReorderElementwiseOpsOnTranspose]
+//===----------------------------------------------------------------------===//
+
+func.func @transpose_extsi(%a : vector<4x2xi8>) -> vector<2x4xi32> {
+  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : vector<4x2xi8> to vector<4x2xi32>
+  // CHECK: vector.transpose %[[EXT]], [1, 0] : vector<4x2xi32> to vector<2x4xi32>
+  %b = vector.transpose %a, [1, 0]: vector<4x2xi8> to vector<2x4xi8>
+  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
+  return %r : vector<2x4xi32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_same_type
+//  CHECK-SAME: (%[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
+//       CHECK:   %[[ADD:.+]] = arith.addf %[[A]], %[[B]] : vector<4x2xf32>
+//       CHECK:   %[[T:.+]] = vector.transpose %[[ADD]], [1, 0]
+//       CHECK:   return %[[T]]
+
+func.func @transpose_elementwise_same_type(%a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xf32> {
+  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %r = arith.addf %at, %bt : vector<2x4xf32>
+  return %r : vector<2x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_diff_operand_types
+//  CHECK-SAME: (%[[COND:.+]]: vector<4x2xi1>, %[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
+//       CHECK:   %[[S:.+]] = arith.select %[[COND]], %[[A]], %[[B]] : vector<4x2xi1>, vector<4x2xf32>
+//       CHECK:   %[[T:.+]] = vector.transpose %[[S]], [1, 0] : vector<4x2xf32> to vector<2x4xf32>
+//       CHECK:   return %[[T]]
+func.func @transpose_elementwise_diff_operand_types(%cond: vector<4x2xi1>, %a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xf32> {
+  %condt = vector.transpose %cond, [1, 0]: vector<4x2xi1> to vector<2x4xi1>
+  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %r = arith.select %condt, %at, %bt : vector<2x4xi1>, vector<2x4xf32>
+  return %r : vector<2x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_diff_operand_result_type
+//  CHECK-SAME: (%[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
+//       CHECK:   %[[CMP:.+]] = arith.cmpf olt, %[[A]], %[[B]] : vector<4x2xf32>
+//       CHECK:   %[[T:.+]] = vector.transpose %[[CMP]], [1, 0] : vector<4x2xi1> to vector<2x4xi1>
+//       CHECK:   return %[[T]]
+func.func @transpose_elementwise_diff_operand_result_type(%a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xi1> {
+  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %r = arith.cmpf olt, %at, %bt : vector<2x4xf32>
+  return %r : vector<2x4xi1>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_splat_constant
+//  CHECK-SAME: (%[[A:.+]]: vector<4x6x3x2xf32>)
+//       CHECK:   %[[B:.+]] = arith.constant dense<5.000000e+00> : vector<4x6x3x2xf32>
+//       CHECK:   %[[ADD:.+]] = arith.addf %[[A]], %[[B]] : vector<4x6x3x2xf32>
+//       CHECK:   %[[T:.+]] = vector.transpose %[[ADD]], [1, 0, 3, 2] : vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
+//       CHECK:   return %[[T:.+]] : vector<6x4x2x3xf32>
+
+func.func @transpose_elementwise_splat_constant(%a : vector<4x6x3x2xf32>) -> vector<6x4x2x3xf32> {
+  %b = arith.constant dense<5.0> : vector<6x4x2x3xf32>
+  %at = vector.transpose %a, [1, 0, 3, 2]: vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
+  %r = arith.addf %at, %b : vector<6x4x2x3xf32>
+  return %r : vector<6x4x2x3xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_diff_map
+//       CHECK:   vector.transpose
+//       CHECK:   vector.transpose
+//       CHECK:   arith.addf
+func.func @transpose_elementwise_diff_map(%a : vector<4x6x3x2xf32>, %b: vector<6x2x4x3xf32>) -> vector<6x4x2x3xf32> {
+  %at = vector.transpose %a, [1, 0, 3, 2]: vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
+  %bt = vector.transpose %b, [0, 2, 1, 3]: vector<6x2x4x3xf32> to vector<6x4x2x3xf32>
+  %r = arith.addf %at, %bt : vector<6x4x2x3xf32>
+  return %r : vector<6x4x2x3xf32>
+}
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/fill-2d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/fill-2d.mlir
index 881e2799b5b06b0..ed4342b7d50afa2 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/fill-2d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/fill-2d.mlir
@@ -6,7 +6,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=entry -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 func.func @entry() {
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
index a8b6457d64becce..a57348a543c3cfa 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
@@ -5,7 +5,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 func.func @matmul_transpose_a(%A : tensor<?x?xf32>, %B : tensor<?x?xf32>, %C : tensor<?x?xf32>) {
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
index 091665223188f6b..79c9fcac70604bf 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
@@ -4,7 +4,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 func.func @matmul(%A : tensor<?x?xf32>, %B : tensor<?x?xf32>, %C : tensor<?x?xf32>) {
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
index 5784ecbbe40141d..78815a38612e960 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
@@ -6,21 +6,15 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib,%mlir_arm_runner_utils | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib,%native_mlir_arm_runner_utils | \
 // RUN: FileCheck %s
 
 /// This is very similar to the SME multi-tile-matmul.mlir test, except that it
 /// tests a mixed i8 to i32 matmul and outer product fusion which fuses 16
 /// outer products (four per tile) into four 4-way outer products.
 
-/// NOTE: QEMU gives incorrect result for SME SMOPA 4-way outer product
-/// instruction (version <= 8.2.0, latest version at time of writing), see:
-/// https://gitlab.com/qemu-project/qemu/-/issues/2083
-/// This test is expected to fail until a fixed version of QEMU can be used.
-
-/// FIXME: Remove the 'XFAIL' below once a fixed QEMU version is available
-/// (and installed on CI buildbot).
-/// XFAIL: *
+// NOTE: QEMU <= 8.2.0 gives incorrect result for SME SMOPA 4-way outer product
+// instruction see: https://gitlab.com/qemu-project/qemu/-/issues/2083.
 
 func.func @matmul_i8_to_i32(%A : tensor<?x?xi8>, %B : tensor<?x?xi8>, %C : tensor<?x?xi32>) {
   %res = linalg.matmul ins(%A, %B: tensor<?x?xi8>, tensor<?x?xi8>)
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
index 03a7d25cffa7677..243f9e5cde9f53d 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
@@ -6,7 +6,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib,%mlir_arm_runner_utils | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib,%native_mlir_arm_runner_utils | \
 // RUN: FileCheck %s
 
 /// This is very similar to the SME matmul.mlir test, except that it uses a tile
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
index 0b84b5a71b39bba..05eee31ccc6fafa 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
@@ -3,7 +3,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 /// This function uses too many tiles! There's only two i16 tiles (ZA0.H and
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
index 56dce1c65401702..31931475525259b 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
@@ -4,7 +4,7 @@
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = conv
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils
+// DEFINE:    -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // RUN: rm -f %t && %{compile} && %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
index 44b555c8c3ad5e5..6099eea1910d959 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt %s -transform-interpreter -test-transform-dialect-erase-schedule -lower-vector-mask -one-shot-bufferize -buffer-deallocation-pipeline -test-lower-to-llvm | \
-// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils | \
+// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
 func.func @entry() {
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
index edb9de922808c08..bcfeef772743b78 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
@@ -4,7 +4,7 @@
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = matmul_f32
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils
+// DEFINE:    -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir
index 7cdb35918c4c0fc..3090e921553c613 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir
@@ -4,7 +4,7 @@
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = reduce_1d_f32
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_native_utils_lib_dir/libmlir_runner_utils%shlibext,%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext
+// DEFINE:    -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir
index bcfe12e374b4e2a..fc0fd40b4d2652c 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir
@@ -4,7 +4,7 @@
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = reduce_2d_f32
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_native_utils_lib_dir/libmlir_runner_utils%shlibext,%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext
+// DEFINE:    -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
index ab4fd0e30d6547b..8cea9184d0f9c56 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
index caa0d6a71ed37a9..c2162ba56367d4b 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
index 7edb76cc8045dd1..9a9c545445ef08f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
index d17e110e2c2d9da..e004fda1f3a259c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
index c2a4e95e7922e33..7f25338d81fea35 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
index 8fe7e08a66d3813..15e0d8615fe56c9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
index d00d4c87f9bd4d7..2653fd393c9b0b6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
index 49f182ddb1d441a..c59993abb16735c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
index cc2a3733c86333b..1eb334750383404 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
index f33a3abc7a5f7e3..9602c198082f86a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/iterator-based-sqsum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/iterator-based-sqsum.mlir
index 6424c35b9c5d5f6..6d03565f8f7b2a0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/iterator-based-sqsum.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/iterator-based-sqsum.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg b/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
index 6e07eb87987a4a5..9caae19726cb114 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
@@ -1,7 +1,7 @@
 import sys
 from lit.llvm import llvm_config
 
-# FIXME: %mlir_native_utils_lib_dir is set incorrectly on Windows
+# No JIT on win32.
 if sys.platform == "win32":
     config.unsupported = True
 
@@ -14,4 +14,3 @@ if config.mlir_run_arm_sve_tests:
 else:
     config.substitutions.append(("%ENABLE_VLA", "false"))
     config.substitutions.append(("%VLA_ARCH_ATTR_OPTIONS", ""))
-    config.substitutions.append(("%mlir_native_utils_lib_dir", config.mlir_lib_dir))
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/padded_sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/padded_sparse_conv_2d.mlir
index 50dd989416e2a0e..f25a71a6cae148c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/padded_sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/padded_sparse_conv_2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/reshape_dot.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/reshape_dot.mlir
index 73dddefb0e4aa5e..b29c4336b40d0fa 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/reshape_dot.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/reshape_dot.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
index 707c6c34d8dce3f..02950fc81228b2e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
index 69be2ee752216a7..795a41797d0cf39 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
index ac5f773d67182cd..a6fb24e266a1876 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir
index efef01155cc784e..24e1e5ba1c770f9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
index 3b5168db23c584f..f4bfb577efd4882 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
index edeffea211717b5..6710ed95e13439f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
index c5d002aa1639175..e5281049b44c9fb 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
index 9deb5cd05fa3b3a..ecd2819ff2fea03 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
index 12132155e7cb35e..ac6f6672fa842b3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
index 087360f7a1ceb70..db32ea9168bbfb9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
index 3f748015c9580ac..183c16fd429e8f2 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
index 2326234bc06ccf0..fe87882ce9ba41e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
index 51c13085cf3e9d3..df348ab5c211ac9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
index 3e46b6d65112f52..2c35ab051a46612 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
index 97e9d1783f67fad..d782a8bfa3b9812 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir
index 00805d198013d50..c218f95c2e53205 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir
index 9150e97e72481bd..20508f980350fc1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
index 429175c1a16457f..892abea7372932b 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
index b23b2dcc173d92f..1d9ca4d4e8fba03 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
index 8fb6704c7f509e9..9624e347bd5ca3f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
index 5de3aa0a2e97130..096a1618df2d87d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
index 66215a340a0b630..d8c3025947bd4d1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
index 0d9722cd37e65d0..8de0f9a5da9a56a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
index a2ec6df392aaa39..a4bcb09cfb8fc5f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
index 531efb4f7f37f46..a93002dfeb4529f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
index e145c4542a7bfc2..51e81225991bb21 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
index 12f8e34500d818a..8844e4f9acf36ef 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
index c16ae0de182039b..191bca746c0684a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir
index 40738a9f7d7f193..046405aa48cb2a4 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
index b41fda19459e659..adede14cfc5e856 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
index 17fab93b9b21787..2d202132b3daa14 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
index 7255649ccd424af..6a1a5821ad9dafc 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
index 6e875de7481ecd2..ed771862e7140bb 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
index 5e021596efea6f7..0792bda162a7618 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
index 93b8eda2c2aec83..a4ed2a6d05fc074 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
index 8a5712d3fa1b109..db3e2381b5c9bff 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir
index aef3a947e4f0bab..2b3cbf28ed014a5 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir
index 63a6d3acd737e2f..75d1941c50ada78 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
index 005398445828169..e81c8f52d6969d4 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
index fc7b82fdecea343..20bf265a70b5176 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
index a81ec172f599d43..d7f9b45bff8e09b 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
index baab6e7598868ff..af6dcdcfeb26350 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
index 12ef94fc2baa215..8c21f14b5abebbe 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
index 416c137a1dc3e74..1682e808b35eeaa 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
index 14fa22a7013478e..e1daaaf9bb1e5b3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
index c76bf2ccfe35e45..f6000ccf34d7121 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
index 770c4f55a280615..ecc351619d12610 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
index b9d1148301dd169..30e7a5f18b7f792 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
index d615cb66c3d08d9..04392ca7a6209d3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
index 683be61be2a201e..506d357fd4a3ced 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
index 8eadc348020f9aa..3539246e1e5a854 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
index 334ffe492952e43..b56d9d4d5a602b6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
index 5415625ff05d6d4..71416ffa6a9065f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
index b48ff9c9df7483c..a53c248eeed494a 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_permute.mlir
index 664a86c7ad58fd8..1df1caf11386dc3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_permute.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
index 7c78bfc362007d1..187ea81fcdb9804 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
index f3c721535e75c5c..2c505ecf29568c2 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
index 4f1e4312d7bc361..ea369338ce09668 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
index 873322929232a75..28f65b10db47f68 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
index fc23fe501fcf1b6..aba1c52a5af3c33 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
index 5da028c3685c4e5..cb6af2daf1c007a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
index 8cbe8fd1ed695ef..624af1fa9f08ee3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir
index 8588ebd98cc918c..c6b42af47834721 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
index 96ec8bad8b55152..bb1ea60bf8d34dd 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir
index 16c1d7df7e89dd8..58a5747116ee220 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir
index 4797fbb8f5319c0..28702976893d80c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
index 317fe0f225ee699..403cbf4cc6fb59f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
index 1536249e60f286e..a90324c3e6d6061 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
index 0682bc6f314fd17..9e0b44ef15dae62 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
index 085b36a368704dd..ebd3c643c65ab96 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
index eecd970e01ac9c7..de6a1e1aa3c1111 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
index c62cdc900b83d90..cb391b7d59793d0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
index 3f0cf70675ba04c..d80255918433872 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
index bd61563b4b2dd3a..85a4e2fea410ada 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
index d96b07a0db331c2..22555dcee8611fc 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
index 11d23d681c8289b..6c0e1f3703ee474 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
index 3117d2539f17f50..47aded8825d99f6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
index ca8bcd7744c8f40..d74722d8ddce891 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
index da87b5cc3c6da7c..3ccbdb36128f956 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir
index 2b2b8536fe39ed5..713e0f5d0abfd2e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
index d1c58bfb6d59efd..2d279995dada1dd 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
index 16a8b50ab08e5c8..3b2629a2cd2ae5d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
@@ -29,7 +30,7 @@
 // Do the same run, but now with  VLA vectorization.
 // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %}
 
-// UNSUPPORTED: target=aarch64{{.*}}
+// UNSUPPORTED: target=aarch64{{.*}}, mlir_arm_emulator
 
 !Filename = !llvm.ptr
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
index f95c163a57c1649..3d6c944cd2223a0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
index 30be587c8f6119c..d9321b926141c4c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
index 748fffc1e637156..6fdbf161b002978 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
@@ -31,7 +32,7 @@
 // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %}
 
 // Current fails for SVE, see https://github.com/llvm/llvm-project/issues/60626
-// UNSUPPORTED: target=aarch64{{.*}}
+// UNSUPPORTED: target=aarch64{{.*}}, mlir_arm_emulator
 
 #SparseVector = #sparse_tensor.encoding<{ map = (d0) -> (d0 : compressed) }>
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
index fe2f2690e860d6e..06f35562941457d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
index a46c3a8d5ef6c6d..de437509c2e9481 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
index 434cc95094645fc..f6d3e5c7ca99463 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
index 3b7760e5052c393..83e965a26b33612 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
index acb7a99a3418016..0d29afb27c5c2a9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
index 10ccf47c3408313..b771f92525e267a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-store-128-bit-tile.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-store-128-bit-tile.mlir
index 41e724844fe4037..f4275fdde3c78c4 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-store-128-bit-tile.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-store-128-bit-tile.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-vertical.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-vertical.mlir
index 68c31ac1dd8e9cf..b7144be08a853f7 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-vertical.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-vertical.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry_point} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/multi-tile-transpose.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/multi-tile-transpose.mlir
index cd48f2a9ebfd895..7f1566d635cbb57 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/multi-tile-transpose.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/multi-tile-transpose.mlir
@@ -2,7 +2,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib,%mlir_arm_runner_utils | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib,%native_mlir_arm_runner_utils | \
 // RUN: FileCheck %s
 
 #transpose = affine_map<(d0, d1) -> (d1, d0)>
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f16f16f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f16f16f32.mlir
index a06ad37b054e45a..886a990125b4c95 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f16f16f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f16f16f32.mlir
@@ -5,7 +5,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd %t \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%mlir_arm_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
index fb6c06cfd69999d..cc05940fb6d02c3 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
@@ -4,7 +4,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd %t \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry_point} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
index b8458606d3f324e..aa5483572dcf941 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
@@ -4,7 +4,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd %t \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme-f64f64 \
 // DEFINE:   -e %{entry_point} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
index 9a353ec2d2f6665..74ddbc63380b70e 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
@@ -3,18 +3,12 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%mlir_arm_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
-// NOTE: QEMU gives incorrect result for SME SMOPA 4-way outer product
-// instruction (version <= 8.2.0, latest version at time of writing), see:
-// https://gitlab.com/qemu-project/qemu/-/issues/2083 This test is expected to
-// fail (CHECK lines are correct) until a fixed version of QEMU can be used.
-
-// FIXME: Remove the 'XFAIL' below once a fixed QEMU version is available
-// (and installed on CI buildbot).
-// XFAIL: *
+// NOTE: QEMU <= 8.2.0 gives incorrect result for SME SMOPA 4-way outer product
+// instruction see: https://gitlab.com/qemu-project/qemu/-/issues/2083.
 
 // NOTE: there is no non-widening variant for these types and this test can't
 // be lowered without the widening pass, therefore we can't check if the result
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/ssve.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/ssve.mlir
index fcf070186359b22..aa8a2aa45d65b32 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/ssve.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/ssve.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=i32 \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // NOTE: To run this test, your CPU must support SME.
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
index 4c02bcb9d192fb9..4a6c6f2ee528bb5 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
@@ -6,7 +6,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%mlir_arm_runner_utils
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/tile-fill.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/tile-fill.mlir
index 3d74508cd23b573..b94345c70c7136f 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/tile-fill.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/tile-fill.mlir
@@ -2,7 +2,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:  -march=aarch64 -mattr=+sve,+sme \
 // RUN:  -e entry -entry-point-result=i32 \
-// RUN:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 // Integration test demonstrating filling a 32-bit element ZA tile with a
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-read-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-read-2d.mlir
index 7421521b96bf92c..77f5a325728b326 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-read-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-read-2d.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
index 2fef705861f28ed..c3bf379cde61721 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transpose.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transpose.mlir
index 177c96f1d8aae6a..ff20f99b63cd1da 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transpose.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transpose.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry_point} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-load-store.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-load-store.mlir
index 48080fd0a26a2ba..6e25bee65f09529 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-load-store.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-load-store.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=i32 \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s --check-prefix=CHECK-ZA0_D
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-ops.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-ops.mlir
index c8c401bed144695..697fb90f6315498 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-ops.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-ops.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=i32 \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/arrays-of-scalable-vectors.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/arrays-of-scalable-vectors.mlir
index afb23e8e5206601..4ce6281d9fd45c6 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/arrays-of-scalable-vectors.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/arrays-of-scalable-vectors.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt %s -convert-vector-to-scf -arm-sve-legalize-vector-storage -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm | \
-// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%native_mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
 /// This tests basic functionality of arrays of scalable vectors, which in MLIR
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
index 79121bf31c26ed5..c9904328763dbce 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
@@ -2,7 +2,7 @@
 // DEFINE:    -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage\
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry} =
-// DEFINE: %{run} = %mcr_aarch64_cmd %t -e=%{entry} -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext
+// DEFINE: %{run} = %mcr_aarch64_cmd %t -e=%{entry} -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%native_mlir_c_runner_utils
 
 // This check whether the files compiles and generates a temporary that will be executed further down.
 // RUN: %{compile}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
index e9f1bbeafacddfc..d3b1fa4bbbc377f 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s -test-lower-to-llvm | \
 // RUN: %mcr_aarch64_cmd -e entry -entry-point-result=void  \
-// RUN:   -shared-libs=%mlir_c_runner_utils,%mlir_arm_runner_utils \
+// RUN:   -shared-libs=%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils \
 // RUN:   -march=aarch64 -mattr=+sve | \
 // RUN: FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/sve.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/sve.mlir
index 824963cdea3b336..40307c5f9da071d 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/sve.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/sve.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt %s -lower-affine -convert-vector-to-scf -convert-scf-to-cf -convert-vector-to-llvm="enable-arm-sve" -finalize-memref-to-llvm -convert-func-to-llvm -convert-arith-to-llvm -canonicalize | \
-// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%native_mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
 // Note: To run this test, your CPU must support SVE
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
index 3ef41b0bf5cf453..c071bb0b17f4476 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
@@ -4,7 +4,7 @@
 // DEFINE: %{compile} = mlir-opt %s -test-lower-to-llvm
 // DEFINE: %{run} = %mcr_aarch64_cmd -march=aarch64 -mattr=+sve \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_c_runner_utils,%mlir_arm_runner_utils
+// DEFINE:  -shared-libs=%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
index 6e25d9d16b8bde5..4f0869d92da6e3c 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
@@ -4,7 +4,7 @@
 // DEFINE: %{compile} = mlir-opt %s -test-lower-to-llvm
 // DEFINE: %{run} = %mcr_aarch64_cmd -march=aarch64 -mattr=+sve \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%mlir_arm_runner_utils
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/lit.local.cfg b/mlir/test/Integration/lit.local.cfg
index 1dc40f986ead147..a73300780f39a82 100644
--- a/mlir/test/Integration/lit.local.cfg
+++ b/mlir/test/Integration/lit.local.cfg
@@ -11,14 +11,8 @@ def configure_aarch64_mcr_cmd():
     # under emulation, the SVE specific RUN lines in the SparseTensor tests
     # will run under emulation.
     if not (config.mlir_run_arm_sve_tests or config.mlir_run_arm_sme_tests):
-        return mcr_cmd
-
-    config.substitutions.append(
-        (
-            "%mlir_native_utils_lib_dir",
-            config.arm_emulator_utils_lib_dir or config.mlir_lib_dir,
-        )
-    )
+        config.substitutions.append(("%mcr_aarch64_cmd", mcr_cmd))
+        return
 
     if config.arm_emulator_executable:
         if config.arm_emulator_mlir_cpu_runner_executable:
@@ -43,10 +37,24 @@ def configure_aarch64_mcr_cmd():
 
         mcr_cmd = f"{emulation_cmd} {mcr_cmd}"
 
-    return mcr_cmd
+    config.substitutions.append(("%mcr_aarch64_cmd", mcr_cmd))
 
+def configure_aarch64_mlir_utils():
+    # Use mlir runtime libs from the location passed by -DARM_EMULATOR_UTILS_LIB_DIR.
+    if config.arm_emulator_utils_lib_dir:
+      base_dir = config.arm_emulator_utils_lib_dir
+    else:
+      base_dir = config.mlir_lib_dir
 
-aarch64_mcr_cmd = configure_aarch64_mcr_cmd()
+    config.substitutions.append(("%native_mlir_runner_utils",
+                                 base_dir + "/libmlir_runner_utils" + config.llvm_shlib_ext))
+    config.substitutions.append(("%native_mlir_c_runner_utils",
+                                 base_dir + "/libmlir_c_runner_utils" + config.llvm_shlib_ext))
+    config.substitutions.append(("%native_mlir_arm_runner_utils",
+                                 base_dir + "/libmlir_arm_runner_utils" + config.llvm_shlib_ext))
+    # Use passed Arm SME ABI routines, if not present default to stubs.
+    config.substitutions.append(("%native_arm_sme_abi_shlib", config.arm_sme_abi_routines_shlib or
+                                 (base_dir + "/libmlir_arm_sme_abi_stubs" + config.llvm_shlib_ext)))
 
 # AArch64 tests will run under emulation if configured at build time by the
 # following CMake options:
@@ -57,4 +65,7 @@ aarch64_mcr_cmd = configure_aarch64_mcr_cmd()
 #                                             support cross-compilation
 # * ARM_EMULATOR_UTILS_LIB_DIR - AArch64 native utilites library to support
 #                                cross-compilation.
-config.substitutions.append(("%mcr_aarch64_cmd", aarch64_mcr_cmd))
+
+# Check for above options and set correct executables and runtime libs
+configure_aarch64_mcr_cmd()
+configure_aarch64_mlir_utils()
diff --git a/mlir/test/Target/Cpp/invalid.mlir b/mlir/test/Target/Cpp/invalid.mlir
index 513371a09cde1d2..4e777239fba6629 100644
--- a/mlir/test/Target/Cpp/invalid.mlir
+++ b/mlir/test/Target/Cpp/invalid.mlir
@@ -80,6 +80,7 @@ func.func @array_as_result(%arg: !emitc.array<4xi8>) -> (!emitc.array<4xi8>) {
 }
 
 // -----
+
 func.func @ptr_to_array() {
   // expected-error@+1 {{cannot emit pointer to array type '!emitc.ptr<!emitc.array<9xi16>>'}}
   %v = "emitc.variable"(){value = #emitc.opaque<"NULL">} : () -> !emitc.ptr<!emitc.array<9xi16>>
diff --git a/mlir/test/Target/Cpp/switch.mlir b/mlir/test/Target/Cpp/switch.mlir
new file mode 100644
index 000000000000000..0f2e716a98f16bd
--- /dev/null
+++ b/mlir/test/Target/Cpp/switch.mlir
@@ -0,0 +1,856 @@
+// RUN: mlir-translate -mlir-to-cpp %s | FileCheck %s -check-prefix=CPP-DEFAULT
+// RUN: mlir-translate -mlir-to-cpp -declare-variables-at-top %s | FileCheck %s -check-prefix=CPP-DECLTOP
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ptrdiff_t() {
+// CPP-DEFAULT:         ptrdiff_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ptrdiff_t() {
+// CPP-DECLTOP:         ptrdiff_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ptrdiff_t() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> !emitc.ptrdiff_t
+
+  emitc.switch %0 : !emitc.ptrdiff_t
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ssize_t() {
+// CPP-DEFAULT:         ssize_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ssize_t() {
+// CPP-DECLTOP:         ssize_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ssize_t() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> !emitc.ssize_t
+
+  emitc.switch %0 : !emitc.ssize_t
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_size_t() {
+// CPP-DEFAULT:         size_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_size_t() {
+// CPP-DECLTOP:         size_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_size_t() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> !emitc.size_t
+
+  emitc.switch %0 : !emitc.size_t
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_index() {
+// CPP-DEFAULT:         size_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_index() {
+// CPP-DECLTOP:         size_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_index() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> index
+
+  emitc.switch %0 : index
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_opaque() {
+// CPP-DEFAULT:         size_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_opaque() {
+// CPP-DECLTOP:         size_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_opaque() {
+  %0 = "emitc.variable"() {value = #emitc.opaque<"1">} 
+  : () -> !emitc.opaque<"size_t">
+
+  emitc.switch %0 : !emitc.opaque<"size_t">
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i1() {
+// CPP-DEFAULT:         bool v1 = true;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i1() {
+// CPP-DECLTOP:         bool v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = true;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i1() {
+  %0 = "emitc.variable"(){value = 1 : i1} : () -> i1
+
+  emitc.switch %0 : i1
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i8() {
+// CPP-DEFAULT:         int8_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i8() {
+// CPP-DECLTOP:         int8_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i8() {
+  %0 = "emitc.variable"(){value = 1 : i8} : () -> i8
+
+  emitc.switch %0 : i8
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ui8() {
+// CPP-DEFAULT:         uint8_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ui8() {
+// CPP-DECLTOP:         uint8_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ui8() {
+  %0 = "emitc.variable"(){value = 1 : ui8} : () -> ui8
+
+  emitc.switch %0 : ui8
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i16() {
+// CPP-DEFAULT:         int16_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i16() {
+// CPP-DECLTOP:         int16_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i16() {
+  %0 = "emitc.variable"(){value = 1 : i16} : () -> i16
+
+  emitc.switch %0 : i16
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ui16() {
+// CPP-DEFAULT:         uint16_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ui16() {
+// CPP-DECLTOP:         uint16_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ui16() {
+  %0 = "emitc.variable"(){value = 1 : ui16} : () -> ui16
+
+  emitc.switch %0 : ui16
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i32() {
+// CPP-DEFAULT:         int32_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i32() {
+// CPP-DECLTOP:         int32_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i32() {
+  %0 = "emitc.variable"(){value = 1 : i32} : () -> i32
+
+  emitc.switch %0 : i32
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ui32() {
+// CPP-DEFAULT:         uint32_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ui32() {
+// CPP-DECLTOP:         uint32_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ui32() {
+  %0 = "emitc.variable"(){value = 1 : ui32} : () -> ui32
+
+  emitc.switch %0 : ui32
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i64() {
+// CPP-DEFAULT:         int64_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i64() {
+// CPP-DECLTOP:         int64_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i64() {
+  %0 = "emitc.variable"(){value = 1 : i64} : () -> i64
+
+  emitc.switch %0 : i64
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ui64() {
+// CPP-DEFAULT:         uint64_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ui64() {
+// CPP-DECLTOP:         uint64_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ui64() {
+  %0 = "emitc.variable"(){value = 1 : ui64} : () -> ui64
+
+  emitc.switch %0 : ui64
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll
index 9c24ed2b75746bf..6bde174642d5400 100644
--- a/mlir/test/Target/LLVMIR/Import/import-failure.ll
+++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll
@@ -359,12 +359,6 @@ declare void @llvm.experimental.noalias.scope.decl(metadata)
 
 ; // -----
 
-; CHECK:      import-failure.ll
-; CHECK-SAME: error: cannot translate data layout: i8:8:8:8
-target datalayout = "e-i8:8:8:8"
-
-; // -----
-
 ; CHECK:      import-failure.ll
 ; CHECK-SAME: warning: unhandled data layout token: ni:42
 target datalayout = "e-ni:42-i64:64"
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 9a5528002ef5e96..0fa82cef0a0f5a5 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -409,6 +409,8 @@ define void @masked_load_store_intrinsics(ptr %vec, <7 x i1> %mask) {
   %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 1, <7 x i1> %mask, <7 x float> undef)
   ; CHECK:  %[[VAL2:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32}
   %2 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1)
+  ; CHECK:  %[[VAL3:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32, nontemporal}
+  %3 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1), !nontemporal !{i32 1}
   ; CHECK:  llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32}
   ; CHECK-SAME:  vector<7xf32>, vector<7xi1> into !llvm.ptr
   call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr %vec, i32 8, <7 x i1> %mask)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 7878aa5ee46d4f0..e2eadf14fc97e96 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -417,8 +417,11 @@ llvm.func @masked_load_store_intrinsics(%A: !llvm.ptr, %mask: vector<7xi1>) {
   // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> poison)
   %a = llvm.intr.masked.load %A, %mask { alignment = 1: i32} :
     (!llvm.ptr, vector<7xi1>) -> vector<7xf32>
+  // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> poison), !nontemporal !1
+  %b = llvm.intr.masked.load %A, %mask { alignment = 1: i32, nontemporal} :
+    (!llvm.ptr, vector<7xi1>) -> vector<7xf32>
   // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
-  %b = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i32} :
+  %c = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i32} :
     (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
   // CHECK: call void @llvm.masked.store.v7f32.p0(<7 x float> %{{.*}}, ptr %0, i32 {{.*}}, <7 x i1> %{{.*}})
   llvm.intr.masked.store %b, %A, %mask { alignment = 1: i32} :
diff --git a/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir b/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
index 65ae98b2a74c6ec..b06ad96f4592c5f 100644
--- a/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
@@ -114,3 +114,45 @@ omp.private {type = firstprivate} @multi_block.privatizer : !llvm.ptr alloc {
   llvm.store %arg2, %arg3 : f32, !llvm.ptr
   omp.yield(%arg3 : !llvm.ptr)
 }
+
+// -----
+
+// Verifies fix for https://github.com/llvm/llvm-project/issues/102935.
+//
+// The issue happens since we previously failed to match MLIR values to their
+// corresponding LLVM values in some cases (e.g. char strings with non-const
+// length).
+llvm.func @non_const_len_char_test(%n: !llvm.ptr {fir.bindc_name = "n"}) {
+  %n_val = llvm.load %n : !llvm.ptr -> i64
+  %orig_alloc = llvm.alloca %n_val x i8 {bindc_name = "str"} : (i64) -> !llvm.ptr
+  %orig_val = llvm.mlir.undef : !llvm.struct<(ptr, i64)>
+  %orig_val_init = llvm.insertvalue %orig_alloc, %orig_val[0] : !llvm.struct<(ptr, i64)>
+  omp.parallel private(@non_const_len_char %orig_val_init -> %priv_arg : !llvm.struct<(ptr, i64)>) {
+    %dummy = llvm.extractvalue %priv_arg[0] : !llvm.struct<(ptr, i64)>
+    omp.terminator
+  }
+  llvm.return
+}
+
+omp.private {type = firstprivate} @non_const_len_char : !llvm.struct<(ptr, i64)> alloc {
+^bb0(%orig_val: !llvm.struct<(ptr, i64)>):
+  %str_len = llvm.extractvalue %orig_val[1] : !llvm.struct<(ptr, i64)>
+  %priv_alloc = llvm.alloca %str_len x i8 {bindc_name = "str", pinned} : (i64) -> !llvm.ptr
+  %priv_val = llvm.mlir.undef : !llvm.struct<(ptr, i64)>
+  %priv_val_init = llvm.insertvalue %priv_alloc, %priv_val[0] : !llvm.struct<(ptr, i64)>
+  omp.yield(%priv_val_init : !llvm.struct<(ptr, i64)>)
+} copy {
+^bb0(%orig_val: !llvm.struct<(ptr, i64)>, %priv_val: !llvm.struct<(ptr, i64)>):
+  llvm.call @foo() : () -> ()
+  omp.yield(%priv_val : !llvm.struct<(ptr, i64)>)
+}
+
+llvm.func @foo()
+
+// CHECK-LABEL: @non_const_len_char_test..omp_par({{.*}})
+// CHECK-NEXT:    omp.par.entry:
+// Verify that we found the privatizer by checking that we properly inlined the
+// bodies of the alloc and copy regions.
+// CHECK:         %[[STR_LEN:.*]] = extractvalue { ptr, i64 } %{{.*}}, 1
+// CHECK:         %{{.*}} = alloca i8, i64 %[[STR_LEN]], align 1
+// CHECK:         call void @foo()
diff --git a/mlir/test/Transforms/test-legalize-target-materialization-no-uses.mlir b/mlir/test/Transforms/test-legalize-target-materialization-no-uses.mlir
index 19dd2b2ee4b2f21..5e694a04af49a8e 100644
--- a/mlir/test/Transforms/test-legalize-target-materialization-no-uses.mlir
+++ b/mlir/test/Transforms/test-legalize-target-materialization-no-uses.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -test-target-materialization-with-no-uses %s | FileCheck %s
+// RUN: mlir-opt -test-target-materialization-with-no-uses -split-input-file %s | FileCheck %s
 
 // The conversion is set up as follows:
 // - type_changer ops are illegal;
@@ -25,3 +25,12 @@ func.func @foo() {
   "test.type_consumer"(%2) : (i64) -> ()
   return
 }
+
+// -----
+
+// CHECK-LABEL: @direct_forward
+func.func @direct_forward(%arg0 : i16) -> i16 {
+  // CHECK-NEXT: return
+  %0 = "test.type_changer"(%arg0) : (i16) -> i16
+  return %0 : i16
+}
diff --git a/mlir/test/Transforms/test-legalize-type-conversion.mlir b/mlir/test/Transforms/test-legalize-type-conversion.mlir
index 252b990210a1809..cf2c9f6a8ec4417 100644
--- a/mlir/test/Transforms/test-legalize-type-conversion.mlir
+++ b/mlir/test/Transforms/test-legalize-type-conversion.mlir
@@ -99,6 +99,7 @@ func.func @test_block_argument_not_converted() {
 func.func @test_signature_conversion_no_converter() {
   "test.signature_conversion_no_converter"() ({
   // expected-error@below {{failed to legalize unresolved materialization from ('f64') to 'f32' that remained live after conversion}}
+  // expected-note@below {{see existing live user here}}
   ^bb0(%arg0: f32):
     "test.type_consumer"(%arg0) : (f32) -> ()
     "test.return"(%arg0) : (f32) -> ()
diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index 29c763b622e877d..72aaa7dc4f89734 100644
--- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -374,27 +374,27 @@ struct TestVectorTransferCollapseInnerMostContiguousDims
   }
 };
 
-struct TestSinkVectorBroadcast
-    : public PassWrapper<TestSinkVectorBroadcast, OperationPass<func::FuncOp>> {
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSinkVectorBroadcast)
+struct TestVectorSinkPatterns
+    : public PassWrapper<TestVectorSinkPatterns, OperationPass<func::FuncOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorSinkPatterns)
 
-  TestSinkVectorBroadcast() = default;
-  TestSinkVectorBroadcast(const TestSinkVectorBroadcast &pass) = default;
+  TestVectorSinkPatterns() = default;
+  TestVectorSinkPatterns(const TestVectorSinkPatterns &pass) = default;
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<memref::MemRefDialect, affine::AffineDialect>();
   }
 
-  StringRef getArgument() const final { return "test-sink-vector-broadcast"; }
+  StringRef getArgument() const final { return "test-vector-sink-patterns"; }
 
   StringRef getDescription() const final {
     return "Test lowering patterns that eliminate redundant brodacast "
-           "operations.";
+           "and transpose operations.";
   }
 
   void runOnOperation() override {
     RewritePatternSet patterns(&getContext());
-    populateSinkVectorBroadcastPatterns(patterns);
+    populateSinkVectorOpsPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
   }
 };
@@ -919,7 +919,7 @@ void registerTestVectorLowerings() {
 
   PassRegistration<TestVectorTransferCollapseInnerMostContiguousDims>();
 
-  PassRegistration<TestSinkVectorBroadcast>();
+  PassRegistration<TestVectorSinkPatterns>();
 
   PassRegistration<TestVectorReduceToContractPatternsPatterns>();
 
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index 411a02849f9dd88..d68d9b1a1f43a27 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -54,6 +54,11 @@ config.mlir_run_cuda_sm80_lt_tests = @MLIR_RUN_CUDA_SM80_LT_TESTS@
 config.mlir_run_cuda_sm90_tests = @MLIR_RUN_CUDA_SM90_TESTS@
 config.mlir_include_integration_tests = @MLIR_INCLUDE_INTEGRATION_TESTS@
 config.arm_emulator_executable = "@ARM_EMULATOR_EXECUTABLE@"
+# Some tests marked with 'UNSUPPORTED: target=aarch64{{.*}}' are still run when
+# configured with ARM_EMULATOR_EXECUTABLE and the default target is not aarch64.
+# Add feature 'mlir_arm_emulator' here and to UNSUPPORTED list of such tests.
+if config.arm_emulator_executable:
+    config.available_features.add("mlir_arm_emulator")
 config.arm_emulator_options = "@ARM_EMULATOR_OPTIONS@"
 config.arm_emulator_mlir_cpu_runner_executable = "@ARM_EMULATOR_MLIR_CPU_RUNNER_EXECUTABLE@"
 config.arm_emulator_lli_executable = "@ARM_EMULATOR_LLI_EXECUTABLE@"
diff --git a/mlir/test/python/dialects/linalg/opdsl/emit_matmul.py b/mlir/test/python/dialects/linalg/opdsl/emit_matmul.py
index 18c237c68081a17..64df4e1276222f8 100644
--- a/mlir/test/python/dialects/linalg/opdsl/emit_matmul.py
+++ b/mlir/test/python/dialects/linalg/opdsl/emit_matmul.py
@@ -63,8 +63,8 @@ def matmul_poly(
             RankedTensorType.get((4, 16), f32), RankedTensorType.get((16, 8), f32)
         )
         def test_matmul_mono(lhs, rhs):
-            init_result = tensor.EmptyOp([4, 8], f32)
-            return matmul_mono(lhs, rhs, outs=[init_result.result])
+            init_result = tensor.empty([4, 8], f32)
+            return matmul_mono(lhs, rhs, outs=[init_result])
 
         # CHECK-LABEL: @test_i8i8i32_matmul
         # CHECK:      ^{{.*}}(%[[A_ARG:.+]]: i8, %[[B_ARG:.+]]: i8, %[[C_ARG:.+]]: i32)
diff --git a/mlir/test/python/dialects/linalg/ops.py b/mlir/test/python/dialects/linalg/ops.py
index b147551c2e73dbd..3bfbcf7d7f7c81d 100644
--- a/mlir/test/python/dialects/linalg/ops.py
+++ b/mlir/test/python/dialects/linalg/ops.py
@@ -97,7 +97,7 @@ def testNamedStructuredOpGenericForm():
                 RankedTensorType.get((4, 16), f32), RankedTensorType.get((16, 8), f32)
             )
             def named_form(lhs, rhs):
-                init_result = tensor.EmptyOp([4, 8], f32)
+                init_result = tensor.empty([4, 8], f32)
                 #      CHECK: "linalg.matmul"(%{{.*}})
                 # CHECK-SAME:    cast = #linalg.type_fn<cast_signed>
                 # CHECK-SAME:    operandSegmentSizes = array<i32: 2, 1>
@@ -106,7 +106,7 @@ def named_form(lhs, rhs):
                 # CHECK-NEXT:    arith.addf{{.*}} (f32, f32) -> f32
                 # CHECK-NEXT:    linalg.yield{{.*}} (f32) -> ()
                 # CHECK-NEXT: (tensor<4x16xf32>, tensor<16x8xf32>, tensor<4x8xf32>) -> tensor<4x8xf32>
-                return linalg.matmul(lhs, rhs, outs=[init_result.result])
+                return linalg.matmul(lhs, rhs, outs=[init_result])
 
     module.operation.print(print_generic_op_form=True)
 
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 284f98875170cdb..aea20c6ec314350 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -11,13 +11,15 @@ add_dependencies(PluginCommon intrinsics_gen)
 
 # Only enable JIT for those targets that LLVM can support.
 set(supported_jit_targets AMDGPU NVPTX)
-foreach(target IN LISTS supported_jit_targets)
-  if("${target}" IN_LIST LLVM_TARGETS_TO_BUILD)
-	  target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${target}")
-    llvm_map_components_to_libnames(llvm_libs ${target})
-    target_link_libraries(PluginCommon PRIVATE ${llvm_libs})
-  endif()
-endforeach()
+if (NOT LLVM_LINK_LLVM_DYLIB)
+  foreach(target IN LISTS supported_jit_targets)
+    if("${target}" IN_LIST LLVM_TARGETS_TO_BUILD)
+      target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${target}")
+      llvm_map_components_to_libnames(llvm_libs ${target})
+      target_link_libraries(PluginCommon PRIVATE ${llvm_libs})
+    endif()
+  endforeach()
+endif()
 
 # Include the RPC server from the `libc` project if availible.
 if(TARGET llvmlibc_rpc_server AND ${LIBOMPTARGET_GPU_LIBC_SUPPORT})
diff --git a/openmp/runtime/src/include/omp.h.var b/openmp/runtime/src/include/omp.h.var
index eb3ab7778606a35..dee46e51d6b340c 100644
--- a/openmp/runtime/src/include/omp.h.var
+++ b/openmp/runtime/src/include/omp.h.var
@@ -476,7 +476,8 @@
     typedef enum omp_pause_resource_t {
       omp_pause_resume = 0,
       omp_pause_soft = 1,
-      omp_pause_hard = 2
+      omp_pause_hard = 2,
+      omp_pause_stop_tool = 3
     } omp_pause_resource_t;
     extern int __KAI_KMPC_CONVENTION omp_pause_resource(omp_pause_resource_t, int);
     extern int __KAI_KMPC_CONVENTION omp_pause_resource_all(omp_pause_resource_t);
diff --git a/openmp/runtime/src/include/omp_lib.F90.var b/openmp/runtime/src/include/omp_lib.F90.var
index 63a3c93b8d92965..5133915c7d8cbd1 100644
--- a/openmp/runtime/src/include/omp_lib.F90.var
+++ b/openmp/runtime/src/include/omp_lib.F90.var
@@ -192,6 +192,7 @@
         integer (kind=omp_pause_resource_kind), parameter, public :: omp_pause_resume = 0
         integer (kind=omp_pause_resource_kind), parameter, public :: omp_pause_soft = 1
         integer (kind=omp_pause_resource_kind), parameter, public :: omp_pause_hard = 2
+        integer (kind=omp_pause_resource_kind), parameter, public :: omp_pause_stop_tool = 3
 
         integer (kind=omp_interop_fr_kind), parameter, public :: omp_ifr_cuda = 1
         integer (kind=omp_interop_fr_kind), parameter, public :: omp_ifr_cuda_driver = 2
diff --git a/openmp/runtime/src/include/omp_lib.h.var b/openmp/runtime/src/include/omp_lib.h.var
index a709a2f298f8c8c..db1dc889d129989 100644
--- a/openmp/runtime/src/include/omp_lib.h.var
+++ b/openmp/runtime/src/include/omp_lib.h.var
@@ -248,6 +248,8 @@
       parameter(omp_pause_soft=1)
       integer(kind=omp_pause_resource_kind)omp_pause_hard
       parameter(omp_pause_hard=2)
+      integer(kind=omp_pause_resource_kind)omp_pause_stop_tool
+      parameter(omp_pause_stop_tool=3)
 
       integer(kind=omp_interop_fr_kind)omp_ifr_cuda
       parameter(omp_ifr_cuda=1)
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 916c1dc25700875..04bf6c3b34dace2 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -4552,7 +4552,8 @@ extern int __kmpc_get_target_offload();
 typedef enum kmp_pause_status_t {
   kmp_not_paused = 0, // status is not paused, or, requesting resume
   kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause
-  kmp_hard_paused = 2 // status is hard-paused, or, requesting hard pause
+  kmp_hard_paused = 2, // status is hard-paused, or, requesting hard pause
+  kmp_stop_tool_paused = 3 // requesting stop_tool pause
 } kmp_pause_status_t;
 
 // This stores the pause state of the runtime
@@ -4731,6 +4732,8 @@ class kmp_safe_raii_file_t {
       : f(nullptr) {
     open(filename, mode, env_var);
   }
+  kmp_safe_raii_file_t(const kmp_safe_raii_file_t &other) = delete;
+  kmp_safe_raii_file_t &operator=(const kmp_safe_raii_file_t &other) = delete;
   ~kmp_safe_raii_file_t() { close(); }
 
   /// Open filename using mode. This is automatically closed in the destructor.
diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index dd48b1ea5c13f9f..cf5cad04eb57d56 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -1970,7 +1970,6 @@ static bool __kmp_affinity_create_hwloc_map(kmp_i18n_id_t *const msg_id) {
             hw_thread.ids[index + 1] = sub_id;
             index--;
           }
-          prev = memory;
         }
         prev = obj;
       }
@@ -4989,7 +4988,7 @@ static void __kmp_aux_affinity_initialize(kmp_affinity_t &affinity) {
   int depth = __kmp_topology->get_depth();
 
   // Create the table of masks, indexed by thread Id.
-  unsigned numUnique;
+  unsigned numUnique = 0;
   int numAddrs = __kmp_topology->get_num_hw_threads();
   // If OMP_PLACES=cores:<attribute> specified, then attempt
   // to make OS Id mask table using those attributes
diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h
index ed24b6faf2f7e8f..9ab2c0cc70d8c6c 100644
--- a/openmp/runtime/src/kmp_affinity.h
+++ b/openmp/runtime/src/kmp_affinity.h
@@ -29,6 +29,8 @@ class KMPHwlocAffinity : public KMPAffinity {
       mask = hwloc_bitmap_alloc();
       this->zero();
     }
+    Mask(const Mask &other) = delete;
+    Mask &operator=(const Mask &other) = delete;
     ~Mask() { hwloc_bitmap_free(mask); }
     void set(int i) override { hwloc_bitmap_set(mask, i); }
     bool is_set(int i) const override { return hwloc_bitmap_isset(mask, i); }
@@ -1271,7 +1273,7 @@ class hierarchy_info {
       leaf. It corresponds to the number of entries in numPerLevel if we exclude
       all but one trailing 1. */
   kmp_uint32 depth;
-  kmp_uint32 base_num_threads;
+  kmp_uint32 base_num_threads = 0;
   enum init_status { initialized = 0, not_initialized = 1, initializing = 2 };
   volatile kmp_int8 uninitialized; // 0=initialized, 1=not initialized,
   // 2=initialization in progress
@@ -1281,8 +1283,8 @@ class hierarchy_info {
       the parent of a node at level i has. For example, if we have a machine
       with 4 packages, 4 cores/package and 2 HT per core, then numPerLevel =
       {2, 4, 4, 1, 1}. All empty levels are set to 1. */
-  kmp_uint32 *numPerLevel;
-  kmp_uint32 *skipPerLevel;
+  kmp_uint32 *numPerLevel = nullptr;
+  kmp_uint32 *skipPerLevel = nullptr;
 
   void deriveLevels() {
     int hier_depth = __kmp_topology->get_depth();
diff --git a/openmp/runtime/src/kmp_barrier.cpp b/openmp/runtime/src/kmp_barrier.cpp
index 658cee594e48d52..d7ef57c608149e7 100644
--- a/openmp/runtime/src/kmp_barrier.cpp
+++ b/openmp/runtime/src/kmp_barrier.cpp
@@ -444,7 +444,8 @@ static void __kmp_dist_barrier_release(
       next_go = my_current_iter + distributedBarrier::MAX_ITERS;
       my_go_index = tid / b->threads_per_go;
       if (this_thr->th.th_used_in_team.load() == 3) {
-        KMP_COMPARE_AND_STORE_ACQ32(&(this_thr->th.th_used_in_team), 3, 1);
+        (void)KMP_COMPARE_AND_STORE_ACQ32(&(this_thr->th.th_used_in_team), 3,
+                                          1);
       }
       // Check if go flag is set
       if (b->go[my_go_index].go.load() != next_go) {
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index b33c16fa79a658f..fdbf9ff45e35495 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -1589,7 +1589,7 @@ void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
   kmp_dyna_lockseq_t lockseq = __kmp_map_hint_to_lock(hint);
   if (*lk == 0) {
     if (KMP_IS_D_LOCK(lockseq)) {
-      KMP_COMPARE_AND_STORE_ACQ32(
+      (void)KMP_COMPARE_AND_STORE_ACQ32(
           (volatile kmp_int32 *)&((kmp_base_tas_lock_t *)crit)->poll, 0,
           KMP_GET_D_TAG(lockseq));
     } else {
@@ -3486,8 +3486,8 @@ __kmp_enter_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid,
   // Check if it is initialized.
   if (*lk == 0) {
     if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
-      KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0,
-                                  KMP_GET_D_TAG(__kmp_user_lock_seq));
+      (void)KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0,
+                                        KMP_GET_D_TAG(__kmp_user_lock_seq));
     } else {
       __kmp_init_indirect_csptr(crit, loc, global_tid,
                                 KMP_GET_I_TAG(__kmp_user_lock_seq));
diff --git a/openmp/runtime/src/kmp_ftn_entry.h b/openmp/runtime/src/kmp_ftn_entry.h
index 713561734c481ab..9bd3ac973b3528b 100644
--- a/openmp/runtime/src/kmp_ftn_entry.h
+++ b/openmp/runtime/src/kmp_ftn_entry.h
@@ -1427,6 +1427,8 @@ int FTN_STDCALL KMP_EXPAND_NAME(FTN_PAUSE_RESOURCE)(kmp_pause_status_t kind,
 #ifdef KMP_STUB
   return 1; // just fail
 #else
+  if (kind == kmp_stop_tool_paused)
+    return 1; // stop_tool must not be specified
   if (device_num == KMP_EXPAND_NAME(FTN_GET_INITIAL_DEVICE)())
     return __kmpc_pause_resource(kind);
   else {
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 5b4391aa125d417..2c8d9304c46bc25 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -1983,8 +1983,8 @@ int __kmp_fork_call(ident_t *loc, int gtid,
 
 #if OMPT_SUPPORT
     ompt_data_t ompt_parallel_data = ompt_data_none;
-    ompt_data_t *parent_task_data;
-    ompt_frame_t *ompt_frame;
+    ompt_data_t *parent_task_data = NULL;
+    ompt_frame_t *ompt_frame = NULL;
     void *return_address = NULL;
 
     if (ompt_enabled.enabled) {
@@ -5765,8 +5765,8 @@ void __kmp_free_team(kmp_root_t *root,
     for (f = 1; f < team->t.t_nproc; ++f) {
       KMP_DEBUG_ASSERT(team->t.t_threads[f]);
       if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
-        KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team),
-                                    1, 2);
+        (void)KMP_COMPARE_AND_STORE_ACQ32(
+            &(team->t.t_threads[f]->th.th_used_in_team), 1, 2);
       }
       __kmp_free_thread(team->t.t_threads[f]);
     }
@@ -9132,7 +9132,8 @@ int __kmp_pause_resource(kmp_pause_status_t level) {
       __kmp_soft_pause();
       return 0;
     }
-  } else if (level == kmp_hard_paused) { // requesting hard pause
+  } else if (level == kmp_hard_paused || level == kmp_stop_tool_paused) {
+    // requesting hard pause or stop_tool pause
     if (__kmp_pause_status != kmp_not_paused) {
       // error message about already being paused
       return 1;
@@ -9220,8 +9221,8 @@ void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads) {
   // to wake it up.
   for (int f = 1; f < new_nthreads; ++f) {
     KMP_DEBUG_ASSERT(team->t.t_threads[f]);
-    KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0,
-                                3);
+    (void)KMP_COMPARE_AND_STORE_ACQ32(
+        &(team->t.t_threads[f]->th.th_used_in_team), 0, 3);
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { // Wake up sleeping threads
       __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid,
                       (kmp_flag_32<false, false> *)NULL);
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 03ce0dd752af156..7edaa8e127e52c0 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -5276,7 +5276,7 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
   switch (sched) {
   case 0: // no schedule clause specified, we can choose the default
     // let's try to schedule (team_size*10) tasks
-    grainsize = thread->th.th_team_nproc * 10;
+    grainsize = thread->th.th_team_nproc * static_cast<kmp_uint64>(10);
     KMP_FALLTHROUGH();
   case 2: // num_tasks provided
     if (grainsize > tc) {
diff --git a/openmp/runtime/src/kmp_wait_release.h b/openmp/runtime/src/kmp_wait_release.h
index 97db68943da7027..9baf280228ee52f 100644
--- a/openmp/runtime/src/kmp_wait_release.h
+++ b/openmp/runtime/src/kmp_wait_release.h
@@ -104,7 +104,8 @@ template <> struct flag_traits<flag_oncore> {
 template <flag_type FlagType> class kmp_flag {
 protected:
   flag_properties t; /**< "Type" of the flag in loc */
-  kmp_info_t *waiting_threads[1]; /**< Threads sleeping on this thread. */
+  /**< Threads sleeping on this thread. */
+  kmp_info_t *waiting_threads[1] = {nullptr};
   kmp_uint32 num_waiting_threads; /**< Num threads sleeping on this thread. */
   std::atomic<bool> *sleepLoc;
 
@@ -140,7 +141,7 @@ template <typename PtrType, flag_type FlagType, bool Sleepable>
 class kmp_flag_native : public kmp_flag<FlagType> {
 protected:
   volatile PtrType *loc;
-  PtrType checker; /**< When flag==checker, it has been released. */
+  PtrType checker = (PtrType)0; /**< When flag==checker, it has been released */
   typedef flag_traits<FlagType> traits_type;
 
 public:
@@ -234,7 +235,7 @@ template <typename PtrType, flag_type FlagType, bool Sleepable>
 class kmp_flag_atomic : public kmp_flag<FlagType> {
 protected:
   std::atomic<PtrType> *loc; /**< Pointer to flag location to wait on */
-  PtrType checker; /**< Flag == checker means it has been released. */
+  PtrType checker = (PtrType)0; /**< Flag==checker means it has been released */
 public:
   typedef flag_traits<FlagType> traits_type;
   typedef PtrType flag_t;
@@ -935,7 +936,8 @@ class kmp_flag_oncore : public kmp_flag_native<kmp_uint64, flag_oncore, false> {
   kmp_uint32 offset; /**< Portion of flag of interest for an operation. */
   bool flag_switch; /**< Indicates a switch in flag location. */
   enum barrier_type bt; /**< Barrier type. */
-  kmp_info_t *this_thr; /**< Thread to redirect to different flag location. */
+  /**< Thread to redirect to different flag location. */
+  kmp_info_t *this_thr = nullptr;
 #if USE_ITT_BUILD
   void *itt_sync_obj; /**< ITT object to pass to new flag location. */
 #endif
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index e07c5ff4fcc47d8..923eea2a563a917 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -104,9 +104,11 @@ static ompt_start_tool_result_t *ompt_start_tool_result = NULL;
 
 #if KMP_OS_WINDOWS
 static HMODULE ompt_tool_module = NULL;
+static HMODULE ompt_archer_module = NULL;
 #define OMPT_DLCLOSE(Lib) FreeLibrary(Lib)
 #else
 static void *ompt_tool_module = NULL;
+static void *ompt_archer_module = NULL;
 #define OMPT_DLCLOSE(Lib) dlclose(Lib)
 #endif
 
@@ -374,6 +376,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
               "Tool was started and is using the OMPT interface.\n");
           OMPT_VERBOSE_INIT_PRINT(
               "----- END LOGGING OF TOOL REGISTRATION -----\n");
+          ompt_archer_module = h;
           return ret;
         }
         OMPT_VERBOSE_INIT_CONTINUED_PRINT(
@@ -381,6 +384,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
       } else {
         OMPT_VERBOSE_INIT_CONTINUED_PRINT("Failed: %s\n", dlerror());
       }
+      OMPT_DLCLOSE(h);
     }
   }
 #endif
@@ -521,6 +525,8 @@ void ompt_fini() {
     }
   }
 
+  if (ompt_archer_module)
+    OMPT_DLCLOSE(ompt_archer_module);
   if (ompt_tool_module)
     OMPT_DLCLOSE(ompt_tool_module);
   memset(&ompt_enabled, 0, sizeof(ompt_enabled));
diff --git a/openmp/runtime/test/ompt/misc/pause_stop_tool.c b/openmp/runtime/test/ompt/misc/pause_stop_tool.c
new file mode 100644
index 000000000000000..c93913793244582
--- /dev/null
+++ b/openmp/runtime/test/ompt/misc/pause_stop_tool.c
@@ -0,0 +1,44 @@
+// RUN: %libomp-compile-and-run | FileCheck %s
+// REQUIRES: ompt
+#include "callback.h"
+#include <omp.h>
+int main() {
+  int x = 0;
+  int ret = 0;
+#pragma omp parallel
+#pragma omp single
+  x++;
+  // Expected to fail; omp_pause_stop_tool must not be specified
+  ret = omp_pause_resource(omp_pause_stop_tool, omp_get_initial_device());
+  printf("omp_pause_resource %s\n", ret ? "failed" : "succeeded");
+#pragma omp parallel
+#pragma omp single
+  x++;
+  // Expected to succeed
+  ret = omp_pause_resource_all(omp_pause_stop_tool);
+  printf("omp_pause_resource_all %s\n", ret ? "failed" : "succeeded");
+#pragma omp parallel
+#pragma omp single
+  x++;
+  printf("x = %d\n", x);
+  return 0;
+
+  // Check if
+  // -- omp_pause_resource/resource_all returns expected code
+  // -- OMPT interface is shut down as expected
+
+  // CHECK-NOT: {{^}}0: Could not register callback
+  // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
+
+  // CHECK: ompt_event_parallel_begin
+  // CHECK: ompt_event_parallel_end
+
+  // CHECK: omp_pause_resource failed
+
+  // CHECK: ompt_event_parallel_begin
+  // CHECK: ompt_event_parallel_end
+
+  // CHECK: omp_pause_resource_all succeeded
+
+  // CHECK-NOT: ompt_event
+}
diff --git a/polly/lib/Support/GICHelper.cpp b/polly/lib/Support/GICHelper.cpp
index 04d422cf9946231..027e0194732f4e6 100644
--- a/polly/lib/Support/GICHelper.cpp
+++ b/polly/lib/Support/GICHelper.cpp
@@ -143,11 +143,11 @@ static void replace(std::string &str, StringRef find, StringRef replace) {
 }
 
 static void makeIslCompatible(std::string &str) {
-  replace(str, ".", "_");
-  replace(str, "\"", "_");
-  replace(str, " ", "__");
-  replace(str, "=>", "TO");
-  replace(str, "+", "_");
+  llvm::replace(str, '.', '_');
+  llvm::replace(str, '\"', '_');
+  replace(str, StringRef(" "), StringRef("__"));
+  replace(str, StringRef("=>"), StringRef("TO"));
+  llvm::replace(str, '+', '_');
 }
 
 std::string polly::getIslCompatibleName(const std::string &Prefix,
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 5aa6b0142b44e5e..945c3d64a991e6d 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -852,14 +852,14 @@ gentbl(
 )
 
 gentbl(
-    name = "ast_interp_opcodes_gen",
+    name = "ast_bytecode_opcodes_gen",
     tbl_outs = [(
         "-gen-clang-opcodes",
-        "lib/AST/Interp/Opcodes.inc",
+        "lib/AST/ByteCode/Opcodes.inc",
     )],
     tblgen = ":clang-tblgen",
-    td_file = "lib/AST/Interp/Opcodes.td",
-    td_srcs = ["lib/AST/Interp/Opcodes.td"],
+    td_file = "lib/AST/ByteCode/Opcodes.td",
+    td_srcs = ["lib/AST/ByteCode/Opcodes.td"],
 )
 
 gentbl(
@@ -922,11 +922,11 @@ cc_library(
     srcs = glob([
         "lib/AST/*.cpp",
         "lib/AST/*.h",
-        "lib/AST/Interp/*.cpp",
-        "lib/AST/Interp/*.h",
+        "lib/AST/ByteCode/*.cpp",
+        "lib/AST/ByteCode/*.h",
     ]) + [
         "lib/AST/AttrDocTable.inc",
-        "lib/AST/Interp/Opcodes.inc",
+        "lib/AST/ByteCode/Opcodes.inc",
     ],
     hdrs = glob([
         "include/clang/AST/*.h",
@@ -942,7 +942,7 @@ cc_library(
         # this library because it contains files that collide easily with system
         # headers such as `CXXABI.h`.
         "-I$(GENDIR)/$(WORKSPACE_ROOT)/clang/lib/AST",
-        "-I$(GENDIR)/$(WORKSPACE_ROOT)/clang/lib/AST/Interp",
+        "-I$(GENDIR)/$(WORKSPACE_ROOT)/clang/lib/AST/ByteCode",
     ],
     textual_hdrs = [
         "include/clang/AST/AttrImpl.inc",
@@ -972,8 +972,8 @@ cc_library(
         ":ast_comment_html_tags_gen",
         ":ast_comment_html_tags_properties_gen",
         ":ast_comment_nodes_gen",
+        ":ast_bytecode_opcodes_gen",
         ":ast_decl_nodes_gen",
-        ":ast_interp_opcodes_gen",
         ":ast_properties_base_gen",
         ":ast_stmt_data_collectors_gen",
         ":ast_stmt_nodes_gen",
diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index b4889bbd46addd8..ee7ea7458b4488f 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -469,7 +469,7 @@ cc_library(
         "include/lldb/Host/posix/*.h",
     ]),
     strip_include_prefix = "include",
-    deps = [":Utility"],
+    deps = [":UtilityHeaders"],
 )
 
 cc_library(
@@ -484,7 +484,7 @@ cc_library(
         "@platforms//os:macos": [],
         "//conditions:default": ["@platforms//:incompatible"],
     }),
-    deps = [":Utility"],
+    deps = [":UtilityHeaders"],
 )
 
 objc_library(
@@ -502,7 +502,7 @@ objc_library(
         ":Headers",
         ":HostMacOSXHeaders",
         ":HostMacOSXPrivateHeaders",
-        ":Utility",
+        ":UtilityHeaders",
         "//llvm:Support",
         "//llvm:TargetParser",
     ],
@@ -564,7 +564,7 @@ cc_library(
     strip_include_prefix = "include",
     deps = [
         ":Headers",
-        ":Utility",
+        ":UtilityHeaders",
         "//llvm:Object",
         "//llvm:Support",
         "//llvm:TargetParser",
@@ -617,7 +617,6 @@ cc_library(
         ":InterpreterHeaders",
         ":SymbolHeaders",
         ":TargetHeaders",
-        ":Utility",
         "//clang:driver",
         "//llvm:Demangle",
         "//llvm:Support",
@@ -679,7 +678,10 @@ cc_library(
     name = "TargetHeaders",
     hdrs = glob(["include/lldb/Target/**/*.h"]),
     strip_include_prefix = "include",
-    deps = [":AppleArm64ExceptionClass"],
+    deps = [
+        ":AppleArm64ExceptionClass",
+        ":BreakpointHeaders",
+    ],
 )
 
 cc_library(
@@ -721,13 +723,21 @@ cc_library(
     deps = [":Headers"],
 )
 
+cc_library(
+    name = "UtilityHeaders",
+    hdrs = glob(["include/lldb/Utility/**/*.h"]),
+    strip_include_prefix = "include",
+)
+
 cc_library(
     name = "Utility",
     srcs = glob(["source/Utility/**/*.cpp"]),
     hdrs = glob(["include/lldb/Utility/**/*.h"]),
     strip_include_prefix = "include",
     deps = [
+        ":CoreHeaders",
         ":Headers",
+        ":TargetHeaders",
         ":UtilityPrivateHeaders",
         "//llvm:BinaryFormat",
         "//llvm:Support",
diff --git a/utils/bazel/llvm-project-overlay/llvm/utils/lit/tests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/utils/lit/tests/BUILD.bazel
index 13f6f815d399504..d89626a6ee9e67c 100644
--- a/utils/bazel/llvm-project-overlay/llvm/utils/lit/tests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/utils/lit/tests/BUILD.bazel
@@ -34,7 +34,14 @@ expand_template(
         ] + glob(["Inputs/**"]),
     )
     for src in glob(
-        ["*/*.py"],
-        exclude = ["Inputs/**"],
+        ["**/*.py"],
+        exclude = [
+            "Inputs/**",
+            "discovery.py",  # TODO: debug and re-enable
+            "max-time.py",
+            "selecting.py",
+            "shtest-recursive-substitution.py",
+            "use-llvm-tool.py",
+        ],
     )
 ]
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 5a7109a75088eef..7f3538d23acfb22 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -6052,9 +6052,11 @@ cc_library(
     srcs = [
         "lib/Conversion/GPUCommon/GPUOpsLowering.cpp",
         "lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp",
+        "lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp",
     ],
     hdrs = [
         "include/mlir/Conversion/GPUCommon/GPUCommonPass.h",
+        "include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h",
         "lib/Conversion/GPUCommon/GPUOpsLowering.h",
     ],
     includes = ["include"],
@@ -6074,6 +6076,7 @@ cc_library(
         ":LLVMCommonConversion",
         ":LLVMDialect",
         ":MemRefDialect",
+        ":SPIRVDialect",
         ":MemRefToLLVM",
         ":Support",
         ":VectorToLLVM",