[𝘀𝗽𝗿] changes introduced through rebaseupstream/users/vitalybuka/spr/main.nfctsan-move-skipinternalframes-into-sanitizer_common

Created using spr 1.3.4

[skip ci]

28 files changed, 1635 insertions, 309 deletions

diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 42c8336e626c..391c47f88bde 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -293,9 +293,8 @@ void BareMetal::addClangTargetOptions(const ArgList &DriverArgs,
 
 void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
                                              ArgStringList &CC1Args) const {
-  if (DriverArgs.hasArg(options::OPT_nostdinc) ||
-      DriverArgs.hasArg(options::OPT_nostdlibinc) ||
-      DriverArgs.hasArg(options::OPT_nostdincxx))
+  if (DriverArgs.hasArg(options::OPT_nostdinc, options::OPT_nostdlibinc,
+                        options::OPT_nostdincxx))
     return;
 
   const Driver &D = getDriver();
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 60c0e3e74204..e469e420f14f 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -5512,6 +5512,14 @@ static void TryOrBuildParenListInitialization(
   } else if (auto *RT = Entity.getType()->getAs<RecordType>()) {
     bool IsUnion = RT->isUnionType();
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    if (RD->isInvalidDecl()) {
+      // Exit early to avoid confusion when processing members.
+      // We do the same for braced list initialization in
+      // `CheckStructUnionTypes`.
+      Sequence.SetFailed(
+          clang::InitializationSequence::FK_ParenthesizedListInitFailed);
+      return;
+    }
 
     if (!IsUnion) {
       for (const CXXBaseSpecifier &Base : RD->bases()) {
diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c
index a2a1996f6643..40dde2e02918 100644
--- a/clang/test/Driver/linker-wrapper-image.c
+++ b/clang/test/Driver/linker-wrapper-image.c
@@ -19,8 +19,8 @@
 //      OPENMP-COFF: @__start_omp_offloading_entries = hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "omp_offloading_entries$OA"
 // OPENMP-COFF-NEXT: @__stop_omp_offloading_entries = hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "omp_offloading_entries$OZ"
 
-//      OPENMP: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}"
-// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr @.omp_offloading.device_image, ptr getelementptr inbounds ([[[SIZE]] x i8], ptr @.omp_offloading.device_image, i64 1, i64 0), ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries }]
+// OPENMP-NEXT: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD\01{{.*}}", section ".llvm.offloading", align 8
+// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr inbounds ([[[BEGIN:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 1, i64 0), ptr getelementptr inbounds ([[[END:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 1, i64 0), ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries }]
 // OPENMP-NEXT: @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 1, ptr @.omp_offloading.device_images, ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries }
 // OPENMP-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @.omp_offloading.descriptor_reg, ptr null }]
 // OPENMP-NEXT: @llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @.omp_offloading.descriptor_unreg, ptr null }]
diff --git a/clang/test/Driver/nostdincxx.cpp b/clang/test/Driver/nostdincxx.cpp
index ef5702a19c03..94d74f230eb1 100644
--- a/clang/test/Driver/nostdincxx.cpp
+++ b/clang/test/Driver/nostdincxx.cpp
@@ -2,6 +2,7 @@
 // RUN: not %clangxx -nostdinc++ %s 2>&1 | FileCheck %s
 // RUN: not %clangxx -nostdlibinc %s 2>&1 | FileCheck %s
 // RUN: not %clangxx --target=x86_64-unknown-unknown-gnu -fsyntax-only -nostdinc -nostdinc++ %s 2>&1 | FileCheck /dev/null --implicit-check-not=-Wunused-command-line-argument
+// RUN: not %clangxx --target=riscv64-unknown-elf -fsyntax-only -nostdinc -nostdinc++ %s 2>&1 | FileCheck /dev/null --implicit-check-not=-Wunused-command-line-argument
 // CHECK: 'vector' file not found
 #include <vector>
 
diff --git a/clang/test/Lexer/case-insensitive-include-absolute.c b/clang/test/Lexer/case-insensitive-include-absolute.c
index 6247e4808c7f..ef9e131c45df 100644
--- a/clang/test/Lexer/case-insensitive-include-absolute.c
+++ b/clang/test/Lexer/case-insensitive-include-absolute.c
@@ -1,8 +1,8 @@
 // REQUIRES: case-insensitive-filesystem
 
 // RUN: rm -rf %t && split-file %s %t
-// RUN: sed "s|DIR|%/t|g" %t/tu.c.in > %t/tu.c
-// RUN: %clang_cc1 -fsyntax-only %t/tu.c 2>&1 | FileCheck %s --DDIR=%/t
+// RUN: sed "s|DIR|%{/t:real}|g" %t/tu.c.in > %t/tu.c
+// RUN: %clang_cc1 -fsyntax-only %t/tu.c 2>&1 | FileCheck %s -DDIR=%{/t:real}
 
 //--- header.h
 //--- tu.c.in
diff --git a/clang/test/SemaCXX/crash-GH76228.cpp b/clang/test/SemaCXX/crash-GH76228.cpp
new file mode 100644
index 000000000000..33a939582312
--- /dev/null
+++ b/clang/test/SemaCXX/crash-GH76228.cpp
@@ -0,0 +1,28 @@
+// RUN: %clang_cc1 -std=c++20 -verify %s
+// Check we don't crash on incomplete members and bases when handling parenthesized initialization.
+class incomplete; // expected-note@-0 3  {{forward declaration of 'incomplete'}}
+struct foo {
+  int a;
+  incomplete b;
+  // expected-error@-1 {{incomplete type}}
+};
+foo a1(0);
+
+struct one_int {
+    int a;
+};
+struct bar : one_int, incomplete {};
+// expected-error@-1 {{incomplete type}}
+bar a2(0);
+
+incomplete a3[3](1,2,3);
+// expected-error@-1 {{incomplete type}}
+
+struct qux : foo {
+};
+qux a4(0);
+
+struct fred {
+    foo a[3];
+};
+fred a5(0);
diff --git a/clang/test/SemaCXX/paren-list-agg-init.cpp b/clang/test/SemaCXX/paren-list-agg-init.cpp
index f60b20e0d465..c1964a5a9eb0 100644
--- a/clang/test/SemaCXX/paren-list-agg-init.cpp
+++ b/clang/test/SemaCXX/paren-list-agg-init.cpp
@@ -289,7 +289,7 @@ int test() {
   // used to crash
   S a(0, 1);
   S b(0);
-  S c(0, 0, 1); // beforecxx20-warning {{aggregate initialization of type 'S' from a parenthesized list of values is a C++20 extension}}
+  S c(0, 0, 1);
 
   S d {0, 1};
   S e {0};
diff --git a/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp b/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp
index f4f500b17357..161374ae5552 100644
--- a/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp
@@ -8,6 +8,7 @@
 
 #include "OffloadWrapper.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Frontend/Offloading/Utility.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -121,19 +122,36 @@ GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs) {
   SmallVector<Constant *, 4u> ImagesInits;
   ImagesInits.reserve(Bufs.size());
   for (ArrayRef<char> Buf : Bufs) {
+    // We embed the full offloading entry so the binary utilities can parse it.
     auto *Data = ConstantDataArray::get(C, Buf);
-    auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant*/ true,
+    auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true,
                                      GlobalVariable::InternalLinkage, Data,
                                      ".omp_offloading.device_image");
     Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
     Image->setSection(".llvm.offloading");
     Image->setAlignment(Align(object::OffloadBinary::getAlignment()));
 
-    auto *Size = ConstantInt::get(getSizeTTy(M), Buf.size());
+    StringRef Binary(Buf.data(), Buf.size());
+    assert(identify_magic(Binary) == file_magic::offload_binary &&
+           "Invalid binary format");
+
+    // The device image struct contains the pointer to the beginning and end of
+    // the image stored inside of the offload binary. There should only be one
+    // of these for each buffer so we parse it out manually.
+    const auto *Header =
+        reinterpret_cast<const object::OffloadBinary::Header *>(
+            Binary.bytes_begin());
+    const auto *Entry = reinterpret_cast<const object::OffloadBinary::Entry *>(
+        Binary.bytes_begin() + Header->EntryOffset);
+
+    auto *Begin = ConstantInt::get(getSizeTTy(M), Entry->ImageOffset);
+    auto *Size =
+        ConstantInt::get(getSizeTTy(M), Entry->ImageOffset + Entry->ImageSize);
+    Constant *ZeroBegin[] = {Zero, Begin};
     Constant *ZeroSize[] = {Zero, Size};
 
     auto *ImageB =
-        ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroZero);
+        ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroBegin);
     auto *ImageE =
         ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroSize);
 
diff --git a/libc/cmake/modules/LLVMLibCObjectRules.cmake b/libc/cmake/modules/LLVMLibCObjectRules.cmake
index 5fbbfd58db2d..c3e3fa2dccfe 100644
--- a/libc/cmake/modules/LLVMLibCObjectRules.cmake
+++ b/libc/cmake/modules/LLVMLibCObjectRules.cmake
@@ -26,7 +26,7 @@ function(_get_common_compile_options output_var flags)
     set(ADD_PREFER_GENERIC_FLAG TRUE)
   endif()
 
-  set(compile_options ${LIBC_COMPILE_OPTIONS_DEFAULT} ${ARGN})
+  set(compile_options ${LIBC_COMPILE_OPTIONS_DEFAULT})
   if(LLVM_COMPILER_IS_GCC_COMPATIBLE)
     list(APPEND compile_options "-fpie")
 
@@ -357,11 +357,8 @@ function(create_object_library fq_target_name)
     set(internal_target_name ${fq_target_name})
   endif()
 
-  _get_common_compile_options(
-    compile_options
-    "${ADD_OBJECT_FLAGS}"
-    ${ADD_OBJECT_COMPILE_OPTIONS}
-  )
+  _get_common_compile_options(compile_options "${ADD_OBJECT_FLAGS}")
+  list(APPEND compile_options ${ADD_OBJECT_COMPILE_OPTIONS})
 
   # GPU builds require special handling for the objects because we want to
   # export several different targets at once, e.g. for both Nvidia and AMD.
@@ -640,11 +637,8 @@ function(create_entrypoint_object fq_target_name)
     set(ADD_ENTRYPOINT_OBJ_CXX_STANDARD ${CMAKE_CXX_STANDARD})
   endif()
 
-  _get_common_compile_options(
-    common_compile_options
-    "${ADD_ENTRYPOINT_OBJ_FLAGS}"
-    ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS}
-  )
+  _get_common_compile_options(common_compile_options "${ADD_ENTRYPOINT_OBJ_FLAGS}")
+  list(APPEND common_compile_options ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS})
   get_fq_deps_list(fq_deps_list ${ADD_ENTRYPOINT_OBJ_DEPENDS})
   set(full_deps_list ${fq_deps_list} libc.src.__support.common)
 
diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/transform_error.mandates.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/transform_error.mandates.verify.cpp
index 965e82a7b403..318435660c36 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.expected/transform_error.mandates.verify.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.expected/transform_error.mandates.verify.cpp
@@ -6,6 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+// https://github.com/llvm/llvm-project/pull/76232 breaks this libc++ test.
+// The fix would be to update this file. The issue is that the CI uses 2
+// versions of Clang-18
+// - An older nightly build as the main compiler
+// - A freshly bootstrap build
+// This means the test can't be used until the nightly build is updated.
+// TODO(mordante) Reenable clang-18.
+// UNSUPPORTED: clang-18
+
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // Test the mandates
diff --git a/libcxx/test/libcxx/utilities/expected/expected.void/transform_error.mandates.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.void/transform_error.mandates.verify.cpp
index 09aa1332e980..d9f65e9f5919 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.void/transform_error.mandates.verify.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.void/transform_error.mandates.verify.cpp
@@ -6,6 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+// https://github.com/llvm/llvm-project/pull/76232 breaks this libc++ test.
+// The fix would be to update this file. The issue is that the CI uses 2
+// versions of Clang-18
+// - An older nightly build as the main compiler
+// - A freshly bootstrap build
+// This means the test can't be used until the nightly build is updated.
+// TODO(mordante) Reenable clang-18.
+// UNSUPPORTED: clang-18
+
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // Test the mandates
diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp
index 647a5a9c9d92..8517d328bd05 100644
--- a/libunwind/src/UnwindCursor.hpp
+++ b/libunwind/src/UnwindCursor.hpp
@@ -12,6 +12,8 @@
 #define __UNWINDCURSOR_HPP__
 
 #include "cet_unwind.h"
+#include <errno.h>
+#include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -990,6 +992,7 @@ private:
     R dummy;
     return stepThroughSigReturn(dummy);
   }
+  bool isReadableAddr(const pint_t addr) const;
 #if defined(_LIBUNWIND_TARGET_AARCH64)
   bool setInfoForSigReturn(Registers_arm64 &);
   int stepThroughSigReturn(Registers_arm64 &);
@@ -2700,20 +2703,12 @@ bool UnwindCursor<A, R>::setInfoForSigReturn(Registers_arm64 &) {
   // [1] https://github.com/torvalds/linux/blob/master/arch/arm64/kernel/vdso/sigreturn.S
   const pint_t pc = static_cast<pint_t>(this->getReg(UNW_REG_IP));
   // The PC might contain an invalid address if the unwind info is bad, so
-  // directly accessing it could cause a segfault. Use process_vm_readv to read
-  // the memory safely instead. process_vm_readv was added in Linux 3.2, and
-  // AArch64 supported was added in Linux 3.7, so the syscall is guaranteed to
-  // be present. Unfortunately, there are Linux AArch64 environments where the
-  // libc wrapper for the syscall might not be present (e.g. Android 5), so call
-  // the syscall directly instead.
-  uint32_t instructions[2];
-  struct iovec local_iov = {&instructions, sizeof instructions};
-  struct iovec remote_iov = {reinterpret_cast<void *>(pc), sizeof instructions};
-  long bytesRead =
-      syscall(SYS_process_vm_readv, getpid(), &local_iov, 1, &remote_iov, 1, 0);
+  // directly accessing it could cause a SIGSEGV.
+  if (!isReadableAddr(pc))
+    return false;
+  auto *instructions = reinterpret_cast<const uint32_t *>(pc);
   // Look for instructions: mov x8, #0x8b; svc #0x0
-  if (bytesRead != sizeof instructions || instructions[0] != 0xd2801168 ||
-      instructions[1] != 0xd4000001)
+  if (instructions[0] != 0xd2801168 || instructions[1] != 0xd4000001)
     return false;
 
   _info = {};
@@ -2762,18 +2757,17 @@ int UnwindCursor<A, R>::stepThroughSigReturn(Registers_arm64 &) {
 template <typename A, typename R>
 bool UnwindCursor<A, R>::setInfoForSigReturn(Registers_riscv &) {
   const pint_t pc = static_cast<pint_t>(getReg(UNW_REG_IP));
-  uint32_t instructions[2];
-  struct iovec local_iov = {&instructions, sizeof instructions};
-  struct iovec remote_iov = {reinterpret_cast<void *>(pc), sizeof instructions};
-  long bytesRead =
-      syscall(SYS_process_vm_readv, getpid(), &local_iov, 1, &remote_iov, 1, 0);
+  // The PC might contain an invalid address if the unwind info is bad, so
+  // directly accessing it could cause a SIGSEGV.
+  if (!isReadableAddr(pc))
+    return false;
+  const auto *instructions = reinterpret_cast<const uint32_t *>(pc);
   // Look for the two instructions used in the sigreturn trampoline
   // __vdso_rt_sigreturn:
   //
   // 0x08b00893 li a7,0x8b
   // 0x00000073 ecall
-  if (bytesRead != sizeof instructions || instructions[0] != 0x08b00893 ||
-      instructions[1] != 0x00000073)
+  if (instructions[0] != 0x08b00893 || instructions[1] != 0x00000073)
     return false;
 
   _info = {};
@@ -2822,13 +2816,11 @@ bool UnwindCursor<A, R>::setInfoForSigReturn(Registers_s390x &) {
   // onto the stack.
   const pint_t pc = static_cast<pint_t>(this->getReg(UNW_REG_IP));
   // The PC might contain an invalid address if the unwind info is bad, so
-  // directly accessing it could cause a segfault. Use process_vm_readv to
-  // read the memory safely instead.
-  uint16_t inst;
-  struct iovec local_iov = {&inst, sizeof inst};
-  struct iovec remote_iov = {reinterpret_cast<void *>(pc), sizeof inst};
-  long bytesRead = process_vm_readv(getpid(), &local_iov, 1, &remote_iov, 1, 0);
-  if (bytesRead == sizeof inst && (inst == 0x0a77 || inst == 0x0aad)) {
+  // directly accessing it could cause a SIGSEGV.
+  if (!isReadableAddr(pc))
+    return false;
+  const auto inst = *reinterpret_cast<const uint16_t *>(pc);
+  if (inst == 0x0a77 || inst == 0x0aad) {
     _info = {};
     _info.start_ip = pc;
     _info.end_ip = pc + 2;
@@ -2974,6 +2966,37 @@ bool UnwindCursor<A, R>::getFunctionName(char *buf, size_t bufLen,
                                          buf, bufLen, offset);
 }
 
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+template <typename A, typename R>
+bool UnwindCursor<A, R>::isReadableAddr(const pint_t addr) const {
+  // We use SYS_rt_sigprocmask, inspired by Abseil's AddressIsReadable.
+
+  const auto sigsetAddr = reinterpret_cast<sigset_t *>(addr);
+  // We have to check that addr is nullptr because sigprocmask allows that
+  // as an argument without failure.
+  if (!sigsetAddr)
+    return false;
+  const auto saveErrno = errno;
+  // We MUST use a raw syscall here, as wrappers may try to access
+  // sigsetAddr which may cause a SIGSEGV. A raw syscall however is
+  // safe. Additionally, we need to pass the kernel_sigset_size, which is
+  // different from libc sizeof(sigset_t). For the majority of architectures,
+  // it's 64 bits (_NSIG), and libc NSIG is _NSIG + 1.
+  const auto kernelSigsetSize = NSIG / 8;
+  [[maybe_unused]] const int Result = syscall(
+      SYS_rt_sigprocmask, /*how=*/~0, sigsetAddr, nullptr, kernelSigsetSize);
+  // Because our "how" is invalid, this syscall should always fail, and our
+  // errno should always be EINVAL or an EFAULT. This relies on the Linux
+  // kernel to check copy_from_user before checking if the "how" argument is
+  // invalid.
+  assert(Result == -1);
+  assert(errno == EFAULT || errno == EINVAL);
+  const auto readable = errno != EFAULT;
+  errno = saveErrno;
+  return readable;
+}
+#endif
+
 #if defined(_LIBUNWIND_USE_CET)
 extern "C" void *__libunwind_cet_get_registers(unw_cursor_t *cursor) {
   AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor;
diff --git a/llvm/cmake/config-ix.cmake b/llvm/cmake/config-ix.cmake
index 3e12213f6e6b..bf1b110245bb 100644
--- a/llvm/cmake/config-ix.cmake
+++ b/llvm/cmake/config-ix.cmake
@@ -639,12 +639,25 @@ if(CMAKE_HOST_APPLE AND APPLE)
     if(NOT CMAKE_XCRUN)
       find_program(CMAKE_XCRUN NAMES xcrun)
     endif()
+
+    # First, check if there's ld-classic, which is ld64 in newer SDKs.
     if(CMAKE_XCRUN)
-      execute_process(COMMAND ${CMAKE_XCRUN} -find ld
+      execute_process(COMMAND ${CMAKE_XCRUN} -find ld-classic
         OUTPUT_VARIABLE LD64_EXECUTABLE
         OUTPUT_STRIP_TRAILING_WHITESPACE)
     else()
-      find_program(LD64_EXECUTABLE NAMES ld DOC "The ld64 linker")
+      find_program(LD64_EXECUTABLE NAMES ld-classic DOC "The ld64 linker")
+    endif()
+
+    # Otherwise look for ld directly.
+    if(NOT LD64_EXECUTABLE)
+        if(CMAKE_XCRUN)
+          execute_process(COMMAND ${CMAKE_XCRUN} -find ld
+            OUTPUT_VARIABLE LD64_EXECUTABLE
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+        else()
+          find_program(LD64_EXECUTABLE NAMES ld DOC "The ld64 linker")
+        endif()
     endif()
   endif()
 
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
index 43ed60d7f977..d25b732fdba3 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
@@ -940,8 +940,13 @@ void DWARFVerifier::verifyDebugLineRows() {
         OS << '\n';
       }
 
-      // Verify file index.
-      if (!LineTable->hasFileAtIndex(Row.File)) {
+      // If the prologue contains no file names and the line table has only one
+      // row, do not verify the file index, this is a line table of an empty
+      // file with an end_sequence, but the DWARF standard sets the file number
+      // to 1 by default, otherwise verify file index.
+      if ((LineTable->Prologue.FileNames.size() ||
+           LineTable->Rows.size() != 1) &&
+          !LineTable->hasFileAtIndex(Row.File)) {
         ++NumDebugLineErrors;
         error() << ".debug_line["
                 << format("0x%08" PRIx64,
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 6b54047020a0..3a3b41fb786c 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -621,6 +621,284 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
   return false;
 }
 
+// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
+// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
+static bool UpgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
+                                                 StringRef Name,
+                                                 Function *&NewFn) {
+  if (Name.starts_with("rbit")) {
+    // '(arm|aarch64).rbit'.
+    NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
+                                      F->arg_begin()->getType());
+    return true;
+  }
+
+  if (Name == "thread.pointer") {
+    // '(arm|aarch64).thread.pointer'.
+    NewFn =
+        Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
+    return true;
+  }
+
+  bool Neon = Name.consume_front("neon.");
+  if (Neon) {
+    // '(arm|aarch64).neon.*'.
+    // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
+    // v16i8 respectively.
+    if (Name.consume_front("bfdot.")) {
+      // (arm|aarch64).neon.bfdot.*'.
+      Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+                             .Cases("v2f32.v8i8", "v4f32.v16i8",
+                                    IsArm ? Intrinsic::arm_neon_bfdot
+                                          : Intrinsic::aarch64_neon_bfdot)
+                             .Default(Intrinsic::not_intrinsic);
+      if (ID != Intrinsic::not_intrinsic) {
+        size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
+        assert((OperandWidth == 64 || OperandWidth == 128) &&
+               "Unexpected operand width");
+        LLVMContext &Ctx = F->getParent()->getContext();
+        std::array<Type *, 2> Tys{
+            {F->getReturnType(),
+             FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
+        NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+        return true;
+      }
+      return false; // No other '(arm|aarch64).neon.bfdot.*'.
+    }
+
+    // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
+    // anymore and accept v8bf16 instead of v16i8.
+    if (Name.consume_front("bfm")) {
+      // (arm|aarch64).neon.bfm*'.
+      if (Name.consume_back(".v4f32.v16i8")) {
+        // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
+        Intrinsic::ID ID =
+            StringSwitch<Intrinsic::ID>(Name)
+                .Case("mla", IsArm ? Intrinsic::arm_neon_bfmmla
+                                   : Intrinsic::aarch64_neon_bfmmla)
+                .Case("lalb", IsArm ? Intrinsic::arm_neon_bfmlalb
+                                    : Intrinsic::aarch64_neon_bfmlalb)
+                .Case("lalt", IsArm ? Intrinsic::arm_neon_bfmlalt
+                                    : Intrinsic::aarch64_neon_bfmlalt)
+                .Default(Intrinsic::not_intrinsic);
+        if (ID != Intrinsic::not_intrinsic) {
+          NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+          return true;
+        }
+        return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
+      }
+      return false; // No other '(arm|aarch64).neon.bfm*.
+    }
+    // Continue on to Aarch64 Neon or Arm Neon.
+  }
+  // Continue on to Arm or Aarch64.
+
+  if (IsArm) {
+    // 'arm.*'.
+    if (Neon) {
+      // 'arm.neon.*'.
+      Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+                             .StartsWith("vclz.", Intrinsic::ctlz)
+                             .StartsWith("vcnt.", Intrinsic::ctpop)
+                             .StartsWith("vqadds.", Intrinsic::sadd_sat)
+                             .StartsWith("vqaddu.", Intrinsic::uadd_sat)
+                             .StartsWith("vqsubs.", Intrinsic::ssub_sat)
+                             .StartsWith("vqsubu.", Intrinsic::usub_sat)
+                             .Default(Intrinsic::not_intrinsic);
+      if (ID != Intrinsic::not_intrinsic) {
+        NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
+                                          F->arg_begin()->getType());
+        return true;
+      }
+
+      if (Name.consume_front("vst")) {
+        // 'arm.neon.vst*'.
+        static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
+        SmallVector<StringRef, 2> Groups;
+        if (vstRegex.match(Name, &Groups)) {
+          static const Intrinsic::ID StoreInts[] = {
+              Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
+              Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
+
+          static const Intrinsic::ID StoreLaneInts[] = {
+              Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
+              Intrinsic::arm_neon_vst4lane};
+
+          auto fArgs = F->getFunctionType()->params();
+          Type *Tys[] = {fArgs[0], fArgs[1]};
+          if (Groups[1].size() == 1)
+            NewFn = Intrinsic::getDeclaration(F->getParent(),
+                                              StoreInts[fArgs.size() - 3], Tys);
+          else
+            NewFn = Intrinsic::getDeclaration(
+                F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
+          return true;
+        }
+        return false; // No other 'arm.neon.vst*'.
+      }
+
+      return false; // No other 'arm.neon.*'.
+    }
+
+    if (Name.consume_front("mve.")) {
+      // 'arm.mve.*'.
+      if (Name == "vctp64") {
+        if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
+          // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
+          // the function and deal with it below in UpgradeIntrinsicCall.
+          rename(F);
+          return true;
+        }
+        return false; // Not 'arm.mve.vctp64'.
+      }
+
+      // These too are changed to accept a v2i1 instead of the old v4i1.
+      if (Name.consume_back(".v4i1")) {
+        // 'arm.mve.*.v4i1'.
+        if (Name.consume_back(".predicated.v2i64.v4i32"))
+          // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
+          return Name == "mull.int" || Name == "vqdmull";
+
+        if (Name.consume_back(".v2i64")) {
+          // 'arm.mve.*.v2i64.v4i1'
+          bool IsGather = Name.consume_front("vldr.gather.");
+          if (IsGather || Name.consume_front("vstr.scatter.")) {
+            if (Name.consume_front("base.")) {
+              // Optional 'wb.' prefix.
+              Name.consume_front("wb.");
+              // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
+              // predicated.v2i64.v2i64.v4i1'.
+              return Name == "predicated.v2i64";
+            }
+
+            if (Name.consume_front("offset.predicated."))
+              return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
+                     Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
+
+            // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
+            return false;
+          }
+
+          return false; // No other 'arm.mve.*.v2i64.v4i1'.
+        }
+        return false; // No other 'arm.mve.*.v4i1'.
+      }
+      return false; // No other 'arm.mve.*'.
+    }
+
+    if (Name.consume_front("cde.vcx")) {
+      // 'arm.cde.vcx*'.
+      if (Name.consume_back(".predicated.v2i64.v4i1"))
+        // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
+        return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
+               Name == "3q" || Name == "3qa";
+
+      return false; // No other 'arm.cde.vcx*'.
+    }
+  } else {
+    // 'aarch64.*'.
+    if (Neon) {
+      // 'aarch64.neon.*'.
+      Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+                             .StartsWith("frintn", Intrinsic::roundeven)
+                             .StartsWith("rbit", Intrinsic::bitreverse)
+                             .Default(Intrinsic::not_intrinsic);
+      if (ID != Intrinsic::not_intrinsic) {
+        NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
+                                          F->arg_begin()->getType());
+        return true;
+      }
+
+      if (Name.starts_with("addp")) {
+        // 'aarch64.neon.addp*'.
+        if (F->arg_size() != 2)
+          return false; // Invalid IR.
+        VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
+        if (Ty && Ty->getElementType()->isFloatingPointTy()) {
+          NewFn = Intrinsic::getDeclaration(F->getParent(),
+                                            Intrinsic::aarch64_neon_faddp, Ty);
+          return true;
+        }
+      }
+      return false; // No other 'aarch64.neon.*'.
+    }
+    if (Name.consume_front("sve.")) {
+      // 'aarch64.sve.*'.
+      if (Name.consume_front("bf")) {
+        if (Name.consume_back(".lane")) {
+          // 'aarch64.sve.bf*.lane'.
+          Intrinsic::ID ID =
+              StringSwitch<Intrinsic::ID>(Name)
+                  .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
+                  .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
+                  .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
+                  .Default(Intrinsic::not_intrinsic);
+          if (ID != Intrinsic::not_intrinsic) {
+            NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+            return true;
+          }
+          return false; // No other 'aarch64.sve.bf*.lane'.
+        }
+        return false; // No other 'aarch64.sve.bf*'.
+      }
+
+      if (Name.consume_front("ld")) {
+        // 'aarch64.sve.ld*'.
+        static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
+        if (LdRegex.match(Name)) {
+          Type *ScalarTy =
+              dyn_cast<VectorType>(F->getReturnType())->getElementType();
+          ElementCount EC = dyn_cast<VectorType>(F->arg_begin()->getType())
+                                ->getElementCount();
+          Type *Ty = VectorType::get(ScalarTy, EC);
+          static const Intrinsic::ID LoadIDs[] = {
+              Intrinsic::aarch64_sve_ld2_sret,
+              Intrinsic::aarch64_sve_ld3_sret,
+              Intrinsic::aarch64_sve_ld4_sret,
+          };
+          NewFn = Intrinsic::getDeclaration(F->getParent(),
+                                            LoadIDs[Name[0] - '2'], Ty);
+          return true;
+        }
+        return false; // No other 'aarch64.sve.ld*'.
+      }
+
+      if (Name.consume_front("tuple.")) {
+        // 'aarch64.sve.tuple.*'.
+        if (Name.starts_with("get")) {
+          // 'aarch64.sve.tuple.get*'.
+          Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
+          NewFn = Intrinsic::getDeclaration(F->getParent(),
+                                            Intrinsic::vector_extract, Tys);
+          return true;
+        }
+
+        if (Name.starts_with("set")) {
+          // 'aarch64.sve.tuple.set*'.
+          auto Args = F->getFunctionType()->params();
+          Type *Tys[] = {Args[0], Args[2], Args[1]};
+          NewFn = Intrinsic::getDeclaration(F->getParent(),
+                                            Intrinsic::vector_insert, Tys);
+          return true;
+        }
+
+        static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
+        if (CreateTupleRegex.match(Name)) {
+          // 'aarch64.sve.tuple.create*'.
+          auto Args = F->getFunctionType()->params();
+          Type *Tys[] = {F->getReturnType(), Args[1]};
+          NewFn = Intrinsic::getDeclaration(F->getParent(),
+                                            Intrinsic::vector_insert, Tys);
+          return true;
+        }
+        return false; // No other 'aarch64.sve.tuple.*'.
+      }
+      return false; // No other 'aarch64.sve.*'.
+    }
+  }
+  return false; // No other 'arm.*', 'aarch64.*'.
+}
+
 static Intrinsic::ID ShouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
   if (Name.consume_front("abs."))
     return StringSwitch<Intrinsic::ID>(Name)
@@ -713,225 +991,12 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
   switch (Name[0]) {
   default: break;
   case 'a': {
-    if (Name.starts_with("arm.rbit") || Name.starts_with("aarch64.rbit")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    if (Name.starts_with("aarch64.neon.frintn")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    if (Name.starts_with("aarch64.neon.rbit")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    if (Name == "aarch64.sve.bfdot.lane") {
-      NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                        Intrinsic::aarch64_sve_bfdot_lane_v2);
-      return true;
-    }
-    if (Name == "aarch64.sve.bfmlalb.lane") {
-      NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                        Intrinsic::aarch64_sve_bfmlalb_lane_v2);
-      return true;
-    }
-    if (Name == "aarch64.sve.bfmlalt.lane") {
-      NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                        Intrinsic::aarch64_sve_bfmlalt_lane_v2);
-      return true;
-    }
-    static const Regex LdRegex("^aarch64\\.sve\\.ld[234](.nxv[a-z0-9]+|$)");
-    if (LdRegex.match(Name)) {
-      Type *ScalarTy =
-          dyn_cast<VectorType>(F->getReturnType())->getElementType();
-      ElementCount EC =
-          dyn_cast<VectorType>(F->arg_begin()->getType())->getElementCount();
-      Type *Ty = VectorType::get(ScalarTy, EC);
-      Intrinsic::ID ID =
-          StringSwitch<Intrinsic::ID>(Name)
-              .StartsWith("aarch64.sve.ld2", Intrinsic::aarch64_sve_ld2_sret)
-              .StartsWith("aarch64.sve.ld3", Intrinsic::aarch64_sve_ld3_sret)
-              .StartsWith("aarch64.sve.ld4", Intrinsic::aarch64_sve_ld4_sret)
-              .Default(Intrinsic::not_intrinsic);
-      NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Ty);
-      return true;
-    }
-    if (Name.starts_with("aarch64.sve.tuple.get")) {
-      Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
-      NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                        Intrinsic::vector_extract, Tys);
-      return true;
-    }
-    if (Name.starts_with("aarch64.sve.tuple.set")) {
-      auto Args = F->getFunctionType()->params();
-      Type *Tys[] = {Args[0], Args[2], Args[1]};
-      NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                        Intrinsic::vector_insert, Tys);
-      return true;
-    }
-    static const Regex CreateTupleRegex(
-        "^aarch64\\.sve\\.tuple\\.create[234](.nxv[a-z0-9]+|$)");
-    if (CreateTupleRegex.match(Name)) {
-      auto Args = F->getFunctionType()->params();
-      Type *Tys[] = {F->getReturnType(), Args[1]};
-      NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                        Intrinsic::vector_insert, Tys);
-      return true;
-    }
-    if (Name.starts_with("arm.neon.vclz")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    if (Name.starts_with("arm.neon.vcnt")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
-    if (vstRegex.match(Name)) {
-      static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
-                                                Intrinsic::arm_neon_vst2,
-                                                Intrinsic::arm_neon_vst3,
-                                                Intrinsic::arm_neon_vst4};
-
-      static const Intrinsic::ID StoreLaneInts[] = {
-        Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
-        Intrinsic::arm_neon_vst4lane
-      };
-
-      auto fArgs = F->getFunctionType()->params();
-      Type *Tys[] = {fArgs[0], fArgs[1]};
-      if (!Name.contains("lane"))
-        NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                          StoreInts[fArgs.size() - 3], Tys);
-      else
-        NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                          StoreLaneInts[fArgs.size() - 5], Tys);
-      return true;
-    }
-    if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
-      return true;
-    }
-    if (Name.starts_with("arm.neon.vqadds.")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    if (Name.starts_with("arm.neon.vqaddu.")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    if (Name.starts_with("arm.neon.vqsubs.")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    if (Name.starts_with("arm.neon.vqsubu.")) {
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
-                                        F->arg_begin()->getType());
-      return true;
-    }
-    if (Name.starts_with("aarch64.neon.addp")) {
-      if (F->arg_size() != 2)
-        break; // Invalid IR.
-      VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
-      if (Ty && Ty->getElementType()->isFloatingPointTy()) {
-        NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                          Intrinsic::aarch64_neon_faddp, Ty);
+    bool IsArm = Name.consume_front("arm.");
+    if (IsArm || Name.consume_front("aarch64.")) {
+      if (UpgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
         return true;
-      }
-    }
-
-    // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
-    // respectively
-    if ((Name.starts_with("arm.neon.bfdot.") ||
-         Name.starts_with("aarch64.neon.bfdot.")) &&
-        Name.ends_with("i8")) {
-      Intrinsic::ID IID =
-          StringSwitch<Intrinsic::ID>(Name)
-              .Cases("arm.neon.bfdot.v2f32.v8i8",
-                     "arm.neon.bfdot.v4f32.v16i8",
-                     Intrinsic::arm_neon_bfdot)
-              .Cases("aarch64.neon.bfdot.v2f32.v8i8",
-                     "aarch64.neon.bfdot.v4f32.v16i8",
-                     Intrinsic::aarch64_neon_bfdot)
-              .Default(Intrinsic::not_intrinsic);
-      if (IID == Intrinsic::not_intrinsic)
-        break;
-
-      size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
-      assert((OperandWidth == 64 || OperandWidth == 128) &&
-             "Unexpected operand width");
-      LLVMContext &Ctx = F->getParent()->getContext();
-      std::array<Type *, 2> Tys {{
-        F->getReturnType(),
-        FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
-      }};
-      NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
-      return true;
-    }
-
-    // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
-    // and accept v8bf16 instead of v16i8
-    if ((Name.starts_with("arm.neon.bfm") ||
-         Name.starts_with("aarch64.neon.bfm")) &&
-        Name.ends_with(".v4f32.v16i8")) {
-      Intrinsic::ID IID =
-          StringSwitch<Intrinsic::ID>(Name)
-              .Case("arm.neon.bfmmla.v4f32.v16i8",
-                    Intrinsic::arm_neon_bfmmla)
-              .Case("arm.neon.bfmlalb.v4f32.v16i8",
-                    Intrinsic::arm_neon_bfmlalb)
-              .Case("arm.neon.bfmlalt.v4f32.v16i8",
-                    Intrinsic::arm_neon_bfmlalt)
-              .Case("aarch64.neon.bfmmla.v4f32.v16i8",
-                    Intrinsic::aarch64_neon_bfmmla)
-              .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
-                    Intrinsic::aarch64_neon_bfmlalb)
-              .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
-                    Intrinsic::aarch64_neon_bfmlalt)
-              .Default(Intrinsic::not_intrinsic);
-      if (IID == Intrinsic::not_intrinsic)
-        break;
-
-      std::array<Type *, 0> Tys;
-      NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
-      return true;
-    }
-
-    if (Name == "arm.mve.vctp64" &&
-        cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
-      // A vctp64 returning a v4i1 is converted to return a v2i1. Rename the
-      // function and deal with it below in UpgradeIntrinsicCall.
-      rename(F);
-      return true;
+      break;
     }
-    // These too are changed to accept a v2i1 insteead of the old v4i1.
-    if (Name == "arm.mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
-        Name == "arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
-        Name == "arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
-        Name == "arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
-        Name ==
-            "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
-        Name == "arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
-        Name == "arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
-        Name == "arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
-        Name ==
-            "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
-        Name == "arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
-        Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" ||
-        Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" ||
-        Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" ||
-        Name == "arm.cde.vcx2qa.predicated.v2i64.v4i1" ||
-        Name == "arm.cde.vcx3q.predicated.v2i64.v4i1" ||
-        Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1")
-      return true;
 
     if (Name.consume_front("amdgcn.")) {
       if (Name == "alignbit") {
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index e845bf38df2d..7f2018b3a199 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3247,6 +3247,25 @@ Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) {
     }
   }
 
+  // Fold switch(zext/sext(X)) into switch(X) if possible.
+  if (match(Cond, m_ZExtOrSExt(m_Value(Op0)))) {
+    bool IsZExt = isa<ZExtInst>(Cond);
+    Type *SrcTy = Op0->getType();
+    unsigned NewWidth = SrcTy->getScalarSizeInBits();
+
+    if (all_of(SI.cases(), [&](const auto &Case) {
+          const APInt &CaseVal = Case.getCaseValue()->getValue();
+          return IsZExt ? CaseVal.isIntN(NewWidth)
+                        : CaseVal.isSignedIntN(NewWidth);
+        })) {
+      for (auto &Case : SI.cases()) {
+        APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth);
+        Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase));
+      }
+      return replaceOperand(SI, 0, Op0);
+    }
+  }
+
   KnownBits Known = computeKnownBits(Cond, 0, &SI);
   unsigned LeadingKnownZeros = Known.countMinLeadingZeros();
   unsigned LeadingKnownOnes = Known.countMinLeadingOnes();
diff --git a/llvm/test/Assembler/autoupgrade-thread-pointer.ll b/llvm/test/Assembler/autoupgrade-thread-pointer.ll
index 2eaabb4e921f..b1ed15a7e4ef 100644
--- a/llvm/test/Assembler/autoupgrade-thread-pointer.ll
+++ b/llvm/test/Assembler/autoupgrade-thread-pointer.ll
@@ -5,14 +5,14 @@ declare ptr @llvm.aarch64.thread.pointer()
 declare ptr @llvm.arm.thread.pointer()
 
 define ptr @test1() {
-; CHECK: test1()
+; CHECK-LABEL: define ptr @test1()
 ; CHECK: call ptr @llvm.thread.pointer()
   %1 = call ptr @llvm.aarch64.thread.pointer()
   ret ptr %1
 }
 
 define ptr @test2() {
-; CHECK: test2()
+; CHECK-LABEL: define ptr @test2()
 ; CHECK: call ptr @llvm.thread.pointer()
   %1 = call ptr @llvm.arm.thread.pointer()
   ret ptr %1
diff --git a/llvm/test/CodeGen/AArch64/selectopt.ll b/llvm/test/CodeGen/AArch64/selectopt.ll
index 8922eba0406c..f2b86dcb9211 100644
--- a/llvm/test/CodeGen/AArch64/selectopt.ll
+++ b/llvm/test/CodeGen/AArch64/selectopt.ll
@@ -268,3 +268,596 @@ if.end87:                                         ; preds = %if.then, %while.bod
 while.end:                                        ; preds = %land.rhs, %if.end87, %entry
   ret void
 }
+
+
+define void @replace_or(ptr nocapture noundef %newst, ptr noundef %t, ptr noundef %h, i64 noundef %c, i64 noundef %rc, i64 noundef %ma, i64 noundef %n) {
+; CHECKOO-LABEL: @replace_or(
+; CHECKOO-NEXT:  entry:
+; CHECKOO-NEXT:    [[T1:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], ptr [[NEWST:%.*]], i64 0, i32 2
+; CHECKOO-NEXT:    store ptr [[T:%.*]], ptr [[T1]], align 8
+; CHECKOO-NEXT:    [[H3:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 3
+; CHECKOO-NEXT:    store ptr [[H:%.*]], ptr [[H3]], align 8
+; CHECKOO-NEXT:    [[ORG_C:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 8
+; CHECKOO-NEXT:    store i64 [[C:%.*]], ptr [[ORG_C]], align 8
+; CHECKOO-NEXT:    [[C6:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 1
+; CHECKOO-NEXT:    store i64 [[C]], ptr [[C6]], align 8
+; CHECKOO-NEXT:    [[FLOW:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 7
+; CHECKOO-NEXT:    store i64 [[RC:%.*]], ptr [[FLOW]], align 8
+; CHECKOO-NEXT:    [[CONV:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECKOO-NEXT:    store i32 [[CONV]], ptr [[NEWST]], align 8
+; CHECKOO-NEXT:    [[FLOW10:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 1, i32 7
+; CHECKOO-NEXT:    [[TMP0:%.*]] = load i64, ptr [[FLOW10]], align 8
+; CHECKOO-NEXT:    [[FLOW12:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 2, i32 7
+; CHECKOO-NEXT:    [[TMP1:%.*]] = load i64, ptr [[FLOW12]], align 8
+; CHECKOO-NEXT:    [[CMP13:%.*]] = icmp sgt i64 [[TMP0]], [[TMP1]]
+; CHECKOO-NEXT:    [[CONV15:%.*]] = select i1 [[CMP13]], i64 2, i64 3
+; CHECKOO-NEXT:    [[CMP16_NOT149:%.*]] = icmp sgt i64 [[CONV15]], [[MA:%.*]]
+; CHECKOO-NEXT:    br i1 [[CMP16_NOT149]], label [[WHILE_END:%.*]], label [[LAND_RHS:%.*]]
+; CHECKOO:       land.rhs:
+; CHECKOO-NEXT:    [[CMP_0151:%.*]] = phi i64 [ [[CMP_1:%.*]], [[IF_END87:%.*]] ], [ [[CONV15]], [[ENTRY:%.*]] ]
+; CHECKOO-NEXT:    [[POS_0150:%.*]] = phi i64 [ [[CMP_0151]], [[IF_END87]] ], [ 1, [[ENTRY]] ]
+; CHECKOO-NEXT:    [[SUB:%.*]] = add nsw i64 [[CMP_0151]], -1
+; CHECKOO-NEXT:    [[FLOW19:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 7
+; CHECKOO-NEXT:    [[TMP2:%.*]] = load i64, ptr [[FLOW19]], align 8
+; CHECKOO-NEXT:    [[CMP20:%.*]] = icmp sgt i64 [[TMP2]], [[RC]]
+; CHECKOO-NEXT:    br i1 [[CMP20]], label [[WHILE_BODY:%.*]], label [[WHILE_END]]
+; CHECKOO:       while.body:
+; CHECKOO-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]]
+; CHECKOO-NEXT:    [[T24:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 2
+; CHECKOO-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[T24]], align 8
+; CHECKOO-NEXT:    [[SUB25:%.*]] = add nsw i64 [[POS_0150]], -1
+; CHECKOO-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]]
+; CHECKOO-NEXT:    [[T27:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 2
+; CHECKOO-NEXT:    store ptr [[TMP3]], ptr [[T27]], align 8
+; CHECKOO-NEXT:    [[H30:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 3
+; CHECKOO-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[H30]], align 8
+; CHECKOO-NEXT:    [[H33:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 3
+; CHECKOO-NEXT:    store ptr [[TMP4]], ptr [[H33]], align 8
+; CHECKOO-NEXT:    [[C36:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 1
+; CHECKOO-NEXT:    [[TMP5:%.*]] = load i64, ptr [[C36]], align 8
+; CHECKOO-NEXT:    [[C39:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 1
+; CHECKOO-NEXT:    store i64 [[TMP5]], ptr [[C39]], align 8
+; CHECKOO-NEXT:    [[ORG_C45:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 8
+; CHECKOO-NEXT:    store i64 [[TMP5]], ptr [[ORG_C45]], align 8
+; CHECKOO-NEXT:    [[FLOW51:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 7
+; CHECKOO-NEXT:    store i64 [[TMP2]], ptr [[FLOW51]], align 8
+; CHECKOO-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX18]], align 8
+; CHECKOO-NEXT:    store i32 [[TMP6]], ptr [[ARRAYIDX26]], align 8
+; CHECKOO-NEXT:    store ptr [[T]], ptr [[T24]], align 8
+; CHECKOO-NEXT:    store ptr [[H]], ptr [[H30]], align 8
+; CHECKOO-NEXT:    store i64 [[C]], ptr [[C36]], align 8
+; CHECKOO-NEXT:    [[ORG_C69:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 8
+; CHECKOO-NEXT:    store i64 [[C]], ptr [[ORG_C69]], align 8
+; CHECKOO-NEXT:    store i64 [[RC]], ptr [[FLOW19]], align 8
+; CHECKOO-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX18]], align 8
+; CHECKOO-NEXT:    [[MUL:%.*]] = shl nsw i64 [[CMP_0151]], 1
+; CHECKOO-NEXT:    [[CMP77_NOT_NOT:%.*]] = icmp slt i64 [[MUL]], [[MA]]
+; CHECKOO-NEXT:    br i1 [[CMP77_NOT_NOT]], label [[IF_THEN:%.*]], label [[IF_END87]]
+; CHECKOO:       if.then:
+; CHECKOO-NEXT:    [[SUB79:%.*]] = add nsw i64 [[MUL]], -1
+; CHECKOO-NEXT:    [[FLOW81:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB79]], i32 7
+; CHECKOO-NEXT:    [[TMP7:%.*]] = load i64, ptr [[FLOW81]], align 8
+; CHECKOO-NEXT:    [[FLOW83:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[MUL]], i32 7
+; CHECKOO-NEXT:    [[TMP8:%.*]] = load i64, ptr [[FLOW83]], align 8
+; CHECKOO-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]]
+; CHECKOO-NEXT:    [[ADD:%.*]] = zext i1 [[CMP84]] to i64
+; CHECKOO-NEXT:    [[SPEC_SELECT:%.*]] = or disjoint i64 [[MUL]], [[ADD]]
+; CHECKOO-NEXT:    br label [[IF_END87]]
+; CHECKOO:       if.end87:
+; CHECKOO-NEXT:    [[CMP_1]] = phi i64 [ [[MUL]], [[WHILE_BODY]] ], [ [[SPEC_SELECT]], [[IF_THEN]] ]
+; CHECKOO-NEXT:    [[CMP16_NOT:%.*]] = icmp sgt i64 [[CMP_1]], [[MA]]
+; CHECKOO-NEXT:    br i1 [[CMP16_NOT]], label [[WHILE_END]], label [[LAND_RHS]]
+; CHECKOO:       while.end:
+; CHECKOO-NEXT:    ret void
+;
+; CHECKII-LABEL: @replace_or(
+; CHECKII-NEXT:  entry:
+; CHECKII-NEXT:    [[T1:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], ptr [[NEWST:%.*]], i64 0, i32 2
+; CHECKII-NEXT:    store ptr [[T:%.*]], ptr [[T1]], align 8
+; CHECKII-NEXT:    [[H3:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 3
+; CHECKII-NEXT:    store ptr [[H:%.*]], ptr [[H3]], align 8
+; CHECKII-NEXT:    [[ORG_C:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 8
+; CHECKII-NEXT:    store i64 [[C:%.*]], ptr [[ORG_C]], align 8
+; CHECKII-NEXT:    [[C6:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 1
+; CHECKII-NEXT:    store i64 [[C]], ptr [[C6]], align 8
+; CHECKII-NEXT:    [[FLOW:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 7
+; CHECKII-NEXT:    store i64 [[RC:%.*]], ptr [[FLOW]], align 8
+; CHECKII-NEXT:    [[CONV:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECKII-NEXT:    store i32 [[CONV]], ptr [[NEWST]], align 8
+; CHECKII-NEXT:    [[FLOW10:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 1, i32 7
+; CHECKII-NEXT:    [[TMP0:%.*]] = load i64, ptr [[FLOW10]], align 8
+; CHECKII-NEXT:    [[FLOW12:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 2, i32 7
+; CHECKII-NEXT:    [[TMP1:%.*]] = load i64, ptr [[FLOW12]], align 8
+; CHECKII-NEXT:    [[CMP13:%.*]] = icmp sgt i64 [[TMP0]], [[TMP1]]
+; CHECKII-NEXT:    [[CONV15:%.*]] = select i1 [[CMP13]], i64 2, i64 3
+; CHECKII-NEXT:    [[CMP16_NOT149:%.*]] = icmp sgt i64 [[CONV15]], [[MA:%.*]]
+; CHECKII-NEXT:    br i1 [[CMP16_NOT149]], label [[WHILE_END:%.*]], label [[LAND_RHS:%.*]]
+; CHECKII:       land.rhs:
+; CHECKII-NEXT:    [[CMP_0151:%.*]] = phi i64 [ [[CMP_1:%.*]], [[IF_END87:%.*]] ], [ [[CONV15]], [[ENTRY:%.*]] ]
+; CHECKII-NEXT:    [[POS_0150:%.*]] = phi i64 [ [[CMP_0151]], [[IF_END87]] ], [ 1, [[ENTRY]] ]
+; CHECKII-NEXT:    [[SUB:%.*]] = add nsw i64 [[CMP_0151]], -1
+; CHECKII-NEXT:    [[FLOW19:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 7
+; CHECKII-NEXT:    [[TMP2:%.*]] = load i64, ptr [[FLOW19]], align 8
+; CHECKII-NEXT:    [[CMP20:%.*]] = icmp sgt i64 [[TMP2]], [[RC]]
+; CHECKII-NEXT:    br i1 [[CMP20]], label [[WHILE_BODY:%.*]], label [[WHILE_END]]
+; CHECKII:       while.body:
+; CHECKII-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]]
+; CHECKII-NEXT:    [[T24:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 2
+; CHECKII-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[T24]], align 8
+; CHECKII-NEXT:    [[SUB25:%.*]] = add nsw i64 [[POS_0150]], -1
+; CHECKII-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]]
+; CHECKII-NEXT:    [[T27:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 2
+; CHECKII-NEXT:    store ptr [[TMP3]], ptr [[T27]], align 8
+; CHECKII-NEXT:    [[H30:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 3
+; CHECKII-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[H30]], align 8
+; CHECKII-NEXT:    [[H33:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 3
+; CHECKII-NEXT:    store ptr [[TMP4]], ptr [[H33]], align 8
+; CHECKII-NEXT:    [[C36:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 1
+; CHECKII-NEXT:    [[TMP5:%.*]] = load i64, ptr [[C36]], align 8
+; CHECKII-NEXT:    [[C39:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 1
+; CHECKII-NEXT:    store i64 [[TMP5]], ptr [[C39]], align 8
+; CHECKII-NEXT:    [[ORG_C45:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 8
+; CHECKII-NEXT:    store i64 [[TMP5]], ptr [[ORG_C45]], align 8
+; CHECKII-NEXT:    [[FLOW51:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 7
+; CHECKII-NEXT:    store i64 [[TMP2]], ptr [[FLOW51]], align 8
+; CHECKII-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX18]], align 8
+; CHECKII-NEXT:    store i32 [[TMP6]], ptr [[ARRAYIDX26]], align 8
+; CHECKII-NEXT:    store ptr [[T]], ptr [[T24]], align 8
+; CHECKII-NEXT:    store ptr [[H]], ptr [[H30]], align 8
+; CHECKII-NEXT:    store i64 [[C]], ptr [[C36]], align 8
+; CHECKII-NEXT:    [[ORG_C69:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 8
+; CHECKII-NEXT:    store i64 [[C]], ptr [[ORG_C69]], align 8
+; CHECKII-NEXT:    store i64 [[RC]], ptr [[FLOW19]], align 8
+; CHECKII-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX18]], align 8
+; CHECKII-NEXT:    [[MUL:%.*]] = shl nsw i64 [[CMP_0151]], 1
+; CHECKII-NEXT:    [[CMP77_NOT_NOT:%.*]] = icmp slt i64 [[MUL]], [[MA]]
+; CHECKII-NEXT:    br i1 [[CMP77_NOT_NOT]], label [[IF_THEN:%.*]], label [[IF_END87]]
+; CHECKII:       if.then:
+; CHECKII-NEXT:    [[SUB79:%.*]] = add nsw i64 [[MUL]], -1
+; CHECKII-NEXT:    [[FLOW81:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB79]], i32 7
+; CHECKII-NEXT:    [[TMP7:%.*]] = load i64, ptr [[FLOW81]], align 8
+; CHECKII-NEXT:    [[FLOW83:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[MUL]], i32 7
+; CHECKII-NEXT:    [[TMP8:%.*]] = load i64, ptr [[FLOW83]], align 8
+; CHECKII-NEXT:    [[CMP84:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]]
+; CHECKII-NEXT:    [[ADD:%.*]] = zext i1 [[CMP84]] to i64
+; CHECKII-NEXT:    [[SPEC_SELECT:%.*]] = or disjoint i64 [[MUL]], [[ADD]]
+; CHECKII-NEXT:    br label [[IF_END87]]
+; CHECKII:       if.end87:
+; CHECKII-NEXT:    [[CMP_1]] = phi i64 [ [[MUL]], [[WHILE_BODY]] ], [ [[SPEC_SELECT]], [[IF_THEN]] ]
+; CHECKII-NEXT:    [[CMP16_NOT:%.*]] = icmp sgt i64 [[CMP_1]], [[MA]]
+; CHECKII-NEXT:    br i1 [[CMP16_NOT]], label [[WHILE_END]], label [[LAND_RHS]]
+; CHECKII:       while.end:
+; CHECKII-NEXT:    ret void
+;
+entry:
+  %t1 = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 2
+  store ptr %t, ptr %t1, align 8
+  %h3 = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 3
+  store ptr %h, ptr %h3, align 8
+  %org_c = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 8
+  store i64 %c, ptr %org_c, align 8
+  %c6 = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 1
+  store i64 %c, ptr %c6, align 8
+  %flow = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 7
+  store i64 %rc, ptr %flow, align 8
+  %conv = trunc i64 %n to i32
+  store i32 %conv, ptr %newst, align 8
+  %flow10 = getelementptr inbounds %struct.st, ptr %newst, i64 1, i32 7
+  %0 = load i64, ptr %flow10, align 8
+  %flow12 = getelementptr inbounds %struct.st, ptr %newst, i64 2, i32 7
+  %1 = load i64, ptr %flow12, align 8
+  %cmp13 = icmp sgt i64 %0, %1
+  %conv15 = select i1 %cmp13, i64 2, i64 3
+  %cmp16.not149 = icmp sgt i64 %conv15, %ma
+  br i1 %cmp16.not149, label %while.end, label %land.rhs
+
+land.rhs:                                         ; preds = %entry, %if.end87
+  %cmp.0151 = phi i64 [ %cmp.1, %if.end87 ], [ %conv15, %entry ]
+  %pos.0150 = phi i64 [ %cmp.0151, %if.end87 ], [ 1, %entry ]
+  %sub = add nsw i64 %cmp.0151, -1
+  %flow19 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 7
+  %2 = load i64, ptr %flow19, align 8
+  %cmp20 = icmp sgt i64 %2, %rc
+  br i1 %cmp20, label %while.body, label %while.end
+
+while.body:                                       ; preds = %land.rhs
+  %arrayidx18 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub
+  %t24 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 2
+  %3 = load ptr, ptr %t24, align 8
+  %sub25 = add nsw i64 %pos.0150, -1
+  %arrayidx26 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25
+  %t27 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 2
+  store ptr %3, ptr %t27, align 8
+  %h30 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 3
+  %4 = load ptr, ptr %h30, align 8
+  %h33 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 3
+  store ptr %4, ptr %h33, align 8
+  %c36 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 1
+  %5 = load i64, ptr %c36, align 8
+  %c39 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 1
+  store i64 %5, ptr %c39, align 8
+  %org_c45 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 8
+  store i64 %5, ptr %org_c45, align 8
+  %flow51 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 7
+  store i64 %2, ptr %flow51, align 8
+  %6 = load i32, ptr %arrayidx18, align 8
+  store i32 %6, ptr %arrayidx26, align 8
+  store ptr %t, ptr %t24, align 8
+  store ptr %h, ptr %h30, align 8
+  store i64 %c, ptr %c36, align 8
+  %org_c69 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 8
+  store i64 %c, ptr %org_c69, align 8
+  store i64 %rc, ptr %flow19, align 8
+  store i32 %conv, ptr %arrayidx18, align 8
+  %mul = shl nsw i64 %cmp.0151, 1
+  %cmp77.not.not = icmp slt i64 %mul, %ma
+  br i1 %cmp77.not.not, label %if.then, label %if.end87
+
+if.then:                                          ; preds = %while.body
+  %sub79 = add nsw i64 %mul, -1
+  %flow81 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub79, i32 7
+  %7 = load i64, ptr %flow81, align 8
+  %flow83 = getelementptr inbounds %struct.st, ptr %newst, i64 %mul, i32 7
+  %8 = load i64, ptr %flow83, align 8
+  %cmp84 = icmp slt i64 %7, %8
+  %add = zext i1 %cmp84 to i64
+  %spec.select = or disjoint i64 %mul, %add
+  br label %if.end87
+
+if.end87:                                         ; preds = %if.then, %while.body
+  %cmp.1 = phi i64 [ %mul, %while.body ], [ %spec.select, %if.then ]
+  %cmp16.not = icmp sgt i64 %cmp.1, %ma
+  br i1 %cmp16.not, label %while.end, label %land.rhs
+
+while.end:                                        ; preds = %if.end87, %land.rhs, %entry
+  ret void
+}
+
+
+; This `or` is not transformed as it is not the last instruction in the block
+define i32 @or_notatendofblock(ptr nocapture noundef %x, i32 noundef %n, ptr nocapture noundef readonly %z) {
+; CHECKOO-LABEL: @or_notatendofblock(
+; CHECKOO-NEXT:  entry:
+; CHECKOO-NEXT:    [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECKOO-NEXT:    br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECKOO:       for.body.preheader:
+; CHECKOO-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECKOO-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECKOO:       for.cond.cleanup:
+; CHECKOO-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ]
+; CHECKOO-NEXT:    ret i32 [[Y_0_LCSSA]]
+; CHECKOO:       for.body:
+; CHECKOO-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ]
+; CHECKOO-NEXT:    [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ]
+; CHECKOO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]]
+; CHECKOO-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECKOO-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]]
+; CHECKOO-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0
+; CHECKOO-NEXT:    br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECKOO:       if.then:
+; CHECKOO-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]]
+; CHECKOO-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECKOO-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP0]], [[TMP1]]
+; CHECKOO-NEXT:    [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]]
+; CHECKOO-NEXT:    [[DIV2:%.*]] = sdiv i32 [[DIV1]], [[TMP1]]
+; CHECKOO-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[DIV2]], 0
+; CHECKOO-NEXT:    [[CONV:%.*]] = zext i1 [[CMP5]] to i32
+; CHECKOO-NEXT:    [[OR1:%.*]] = or i32 [[CONV]], [[ADD]]
+; CHECKOO-NEXT:    [[OR:%.*]] = add i32 [[OR1]], 1
+; CHECKOO-NEXT:    br label [[IF_END]]
+; CHECKOO:       if.end:
+; CHECKOO-NEXT:    [[Y_1]] = phi i32 [ [[OR]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
+; CHECKOO-NEXT:    store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4
+; CHECKOO-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECKOO-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECKOO-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+;
+; CHECKII-LABEL: @or_notatendofblock(
+; CHECKII-NEXT:  entry:
+; CHECKII-NEXT:    [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECKII-NEXT:    br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECKII:       for.body.preheader:
+; CHECKII-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECKII-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECKII:       for.cond.cleanup:
+; CHECKII-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ]
+; CHECKII-NEXT:    ret i32 [[Y_0_LCSSA]]
+; CHECKII:       for.body:
+; CHECKII-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ]
+; CHECKII-NEXT:    [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ]
+; CHECKII-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]]
+; CHECKII-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECKII-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]]
+; CHECKII-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0
+; CHECKII-NEXT:    br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECKII:       if.then:
+; CHECKII-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]]
+; CHECKII-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECKII-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP0]], [[TMP1]]
+; CHECKII-NEXT:    [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]]
+; CHECKII-NEXT:    [[DIV2:%.*]] = sdiv i32 [[DIV1]], [[TMP1]]
+; CHECKII-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[DIV2]], 0
+; CHECKII-NEXT:    [[CONV:%.*]] = zext i1 [[CMP5]] to i32
+; CHECKII-NEXT:    [[OR1:%.*]] = or i32 [[CONV]], [[ADD]]
+; CHECKII-NEXT:    [[OR:%.*]] = add i32 [[OR1]], 1
+; CHECKII-NEXT:    br label [[IF_END]]
+; CHECKII:       if.end:
+; CHECKII-NEXT:    [[Y_1]] = phi i32 [ [[OR]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
+; CHECKII-NEXT:    store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4
+; CHECKII-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECKII-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECKII-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+;
+entry:
+  %cmp19 = icmp sgt i32 %n, 0
+  br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %wide.trip.count = zext nneg i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup:
+  %y.0.lcssa = phi i32 [ 0, %entry ], [ %y.1, %if.end ]
+  ret i32 %y.0.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %if.end ]
+  %y.020 = phi i32 [ 0, %for.body.preheader ], [ %y.1, %if.end ]
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %indvars.iv
+  %0 = load i32, ptr %arrayidx, align 4
+  %add = add nsw i32 %0, %y.020
+  %tobool.not = icmp eq i32 %add, 0
+  br i1 %tobool.not, label %if.end, label %if.then
+
+if.then:
+  %arrayidx4 = getelementptr inbounds i32, ptr %z, i64 %indvars.iv
+  %1 = load i32, ptr %arrayidx4, align 4
+  %div = sdiv i32 %0, %1
+  %div1 = sdiv i32 %div, %1
+  %div2 = sdiv i32 %div1, %1
+  %cmp5 = icmp sgt i32 %div2, 0
+  %conv = zext i1 %cmp5 to i32
+  %or1 = or i32 %conv, %add
+  %or = add i32 %or1, 1
+  br label %if.end
+
+if.end:
+  %y.1 = phi i32 [ %or, %if.then ], [ 0, %for.body ]
+  store i32 %y.1, ptr %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+; Similar to the last test, an artificial test with the or as the last instruction and a select in the same group.
+define i32 @or_samegroup(ptr nocapture noundef %x, i32 noundef %n, ptr nocapture noundef readonly %z) {
+; CHECKOO-LABEL: @or_samegroup(
+; CHECKOO-NEXT:  entry:
+; CHECKOO-NEXT:    [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECKOO-NEXT:    br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECKOO:       for.body.preheader:
+; CHECKOO-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECKOO-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECKOO:       for.cond.cleanup:
+; CHECKOO-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ]
+; CHECKOO-NEXT:    ret i32 [[Y_0_LCSSA]]
+; CHECKOO:       for.body:
+; CHECKOO-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ]
+; CHECKOO-NEXT:    [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ]
+; CHECKOO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]]
+; CHECKOO-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECKOO-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]]
+; CHECKOO-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0
+; CHECKOO-NEXT:    br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECKOO:       if.then:
+; CHECKOO-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]]
+; CHECKOO-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECKOO-NEXT:    [[DIV:%.*]] = sdiv i32 [[Y_020]], [[TMP1]]
+; CHECKOO-NEXT:    [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]]
+; CHECKOO-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[DIV1]], 0
+; CHECKOO-NEXT:    [[CONV:%.*]] = zext i1 [[CMP5]] to i32
+; CHECKOO-NEXT:    [[SEL_FROZEN:%.*]] = freeze i1 [[CMP5]]
+; CHECKOO-NEXT:    br i1 [[SEL_FROZEN]], label [[SELECT_END:%.*]], label [[SELECT_FALSE:%.*]]
+; CHECKOO:       select.false:
+; CHECKOO-NEXT:    br label [[SELECT_END]]
+; CHECKOO:       select.end:
+; CHECKOO-NEXT:    [[SEL:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ 1, [[SELECT_FALSE]] ]
+; CHECKOO-NEXT:    [[OR:%.*]] = or i32 [[CONV]], [[SEL]]
+; CHECKOO-NEXT:    br label [[IF_END]]
+; CHECKOO:       if.end:
+; CHECKOO-NEXT:    [[Y_1]] = phi i32 [ [[SEL]], [[SELECT_END]] ], [ 0, [[FOR_BODY]] ]
+; CHECKOO-NEXT:    store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4
+; CHECKOO-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECKOO-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECKOO-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+;
+; CHECKII-LABEL: @or_samegroup(
+; CHECKII-NEXT:  entry:
+; CHECKII-NEXT:    [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECKII-NEXT:    br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECKII:       for.body.preheader:
+; CHECKII-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECKII-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECKII:       for.cond.cleanup:
+; CHECKII-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ]
+; CHECKII-NEXT:    ret i32 [[Y_0_LCSSA]]
+; CHECKII:       for.body:
+; CHECKII-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ]
+; CHECKII-NEXT:    [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ]
+; CHECKII-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]]
+; CHECKII-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECKII-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]]
+; CHECKII-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0
+; CHECKII-NEXT:    br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECKII:       if.then:
+; CHECKII-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]]
+; CHECKII-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECKII-NEXT:    [[DIV:%.*]] = sdiv i32 [[Y_020]], [[TMP1]]
+; CHECKII-NEXT:    [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]]
+; CHECKII-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[DIV1]], 0
+; CHECKII-NEXT:    [[CONV:%.*]] = zext i1 [[CMP5]] to i32
+; CHECKII-NEXT:    [[SEL:%.*]] = select i1 [[CMP5]], i32 [[ADD]], i32 1
+; CHECKII-NEXT:    [[OR:%.*]] = or i32 [[CONV]], [[SEL]]
+; CHECKII-NEXT:    br label [[IF_END]]
+; CHECKII:       if.end:
+; CHECKII-NEXT:    [[Y_1]] = phi i32 [ [[SEL]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
+; CHECKII-NEXT:    store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4
+; CHECKII-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECKII-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECKII-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+;
+entry:
+  %cmp19 = icmp sgt i32 %n, 0
+  br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %wide.trip.count = zext nneg i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup:
+  %y.0.lcssa = phi i32 [ 0, %entry ], [ %y.1, %if.end ]
+  ret i32 %y.0.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %if.end ]
+  %y.020 = phi i32 [ 0, %for.body.preheader ], [ %y.1, %if.end ]
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %indvars.iv
+  %0 = load i32, ptr %arrayidx, align 4
+  %add = add nsw i32 %0, %y.020
+  %tobool.not = icmp eq i32 %add, 0
+  br i1 %tobool.not, label %if.end, label %if.then
+
+if.then:
+  %arrayidx4 = getelementptr inbounds i32, ptr %z, i64 %indvars.iv
+  %1 = load i32, ptr %arrayidx4, align 4
+  %div = sdiv i32 %y.020, %1
+  %div1 = sdiv i32 %div, %1
+  %cmp5 = icmp sgt i32 %div1, 0
+  %conv = zext i1 %cmp5 to i32
+  %sel = select i1 %cmp5, i32 %add, i32 1
+  %or = or i32 %conv, %sel
+  br label %if.end
+
+if.end:
+  %y.1 = phi i32 [ %sel, %if.then ], [ 0, %for.body ]
+  store i32 %y.1, ptr %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Same test again with a one use value group of values on the or
+define i32 @or_oneusevalues(ptr nocapture noundef %x, i32 noundef %n, ptr nocapture noundef readonly %z) {
+; CHECKOO-LABEL: @or_oneusevalues(
+; CHECKOO-NEXT:  entry:
+; CHECKOO-NEXT:    [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECKOO-NEXT:    br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECKOO:       for.body.preheader:
+; CHECKOO-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECKOO-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECKOO:       for.cond.cleanup:
+; CHECKOO-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ]
+; CHECKOO-NEXT:    ret i32 [[Y_0_LCSSA]]
+; CHECKOO:       for.body:
+; CHECKOO-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ]
+; CHECKOO-NEXT:    [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ]
+; CHECKOO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]]
+; CHECKOO-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECKOO-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]]
+; CHECKOO-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0
+; CHECKOO-NEXT:    br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECKOO:       if.then:
+; CHECKOO-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]]
+; CHECKOO-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECKOO-NEXT:    [[DIV:%.*]] = sdiv i32 [[Y_020]], [[TMP1]]
+; CHECKOO-NEXT:    [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]]
+; CHECKOO-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[DIV1]], 0
+; CHECKOO-NEXT:    [[CONV:%.*]] = zext i1 [[CMP5]] to i32
+; CHECKOO-NEXT:    [[ADD1:%.*]] = add i32 [[ADD]], 1
+; CHECKOO-NEXT:    [[ADD2:%.*]] = or i32 [[ADD1]], 1
+; CHECKOO-NEXT:    [[OR:%.*]] = or i32 [[CONV]], [[ADD2]]
+; CHECKOO-NEXT:    br label [[IF_END]]
+; CHECKOO:       if.end:
+; CHECKOO-NEXT:    [[Y_1]] = phi i32 [ [[OR]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
+; CHECKOO-NEXT:    store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4
+; CHECKOO-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECKOO-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECKOO-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+;
+; CHECKII-LABEL: @or_oneusevalues(
+; CHECKII-NEXT:  entry:
+; CHECKII-NEXT:    [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECKII-NEXT:    br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECKII:       for.body.preheader:
+; CHECKII-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECKII-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECKII:       for.cond.cleanup:
+; CHECKII-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ]
+; CHECKII-NEXT:    ret i32 [[Y_0_LCSSA]]
+; CHECKII:       for.body:
+; CHECKII-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ]
+; CHECKII-NEXT:    [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ]
+; CHECKII-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]]
+; CHECKII-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECKII-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]]
+; CHECKII-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0
+; CHECKII-NEXT:    br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]]
+; CHECKII:       if.then:
+; CHECKII-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]]
+; CHECKII-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECKII-NEXT:    [[DIV:%.*]] = sdiv i32 [[Y_020]], [[TMP1]]
+; CHECKII-NEXT:    [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]]
+; CHECKII-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[DIV1]], 0
+; CHECKII-NEXT:    [[CONV:%.*]] = zext i1 [[CMP5]] to i32
+; CHECKII-NEXT:    [[ADD1:%.*]] = add i32 [[ADD]], 1
+; CHECKII-NEXT:    [[ADD2:%.*]] = or i32 [[ADD1]], 1
+; CHECKII-NEXT:    [[OR:%.*]] = or i32 [[CONV]], [[ADD2]]
+; CHECKII-NEXT:    br label [[IF_END]]
+; CHECKII:       if.end:
+; CHECKII-NEXT:    [[Y_1]] = phi i32 [ [[OR]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
+; CHECKII-NEXT:    store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4
+; CHECKII-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECKII-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECKII-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+;
+entry:
+  %cmp19 = icmp sgt i32 %n, 0
+  br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %wide.trip.count = zext nneg i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup:
+  %y.0.lcssa = phi i32 [ 0, %entry ], [ %y.1, %if.end ]
+  ret i32 %y.0.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %if.end ]
+  %y.020 = phi i32 [ 0, %for.body.preheader ], [ %y.1, %if.end ]
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %indvars.iv
+  %0 = load i32, ptr %arrayidx, align 4
+  %add = add nsw i32 %0, %y.020
+  %tobool.not = icmp eq i32 %add, 0
+  br i1 %tobool.not, label %if.end, label %if.then
+
+if.then:
+  %arrayidx4 = getelementptr inbounds i32, ptr %z, i64 %indvars.iv
+  %1 = load i32, ptr %arrayidx4, align 4
+  %div = sdiv i32 %y.020, %1
+  %div1 = sdiv i32 %div, %1
+  %cmp5 = icmp sgt i32 %div1, 0
+  %conv = zext i1 %cmp5 to i32
+  %add1 = add i32 %add, 1
+  %add2 = or i32 %add1, 1
+  %or = or i32 %conv, %add2
+  br label %if.end
+
+if.end:
+  %y.1 = phi i32 [ %or, %if.then ], [ 0, %for.body ]
+  store i32 %y.1, ptr %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/InstCombine/phi.ll b/llvm/test/Transforms/InstCombine/phi.ll
index 90818771675b..717f4c682a15 100644
--- a/llvm/test/Transforms/InstCombine/phi.ll
+++ b/llvm/test/Transforms/InstCombine/phi.ll
@@ -996,10 +996,9 @@ done:
 define i1 @PR24766(i8 %x1, i8 %x2, i8 %condition) {
 ; CHECK-LABEL: @PR24766(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[CONDITION:%.*]] to i32
-; CHECK-NEXT:    switch i32 [[CONV]], label [[EPILOG:%.*]] [
-; CHECK-NEXT:      i32 0, label [[SW1:%.*]]
-; CHECK-NEXT:      i32 1, label [[SW2:%.*]]
+; CHECK-NEXT:    switch i8 [[CONDITION:%.*]], label [[EPILOG:%.*]] [
+; CHECK-NEXT:      i8 0, label [[SW1:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw1:
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[X1:%.*]], [[X2:%.*]]
@@ -1040,10 +1039,9 @@ epilog:
 define i1 @PR24766_no_constants(i8 %x1, i8 %x2, i8 %condition, i1 %another_condition) {
 ; CHECK-LABEL: @PR24766_no_constants(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[CONDITION:%.*]] to i32
-; CHECK-NEXT:    switch i32 [[CONV]], label [[EPILOG:%.*]] [
-; CHECK-NEXT:      i32 0, label [[SW1:%.*]]
-; CHECK-NEXT:      i32 1, label [[SW2:%.*]]
+; CHECK-NEXT:    switch i8 [[CONDITION:%.*]], label [[EPILOG:%.*]] [
+; CHECK-NEXT:      i8 0, label [[SW1:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw1:
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[X1:%.*]], [[X2:%.*]]
@@ -1085,10 +1083,9 @@ epilog:
 define i1 @PR24766_two_constants(i8 %x1, i8 %x2, i8 %condition) {
 ; CHECK-LABEL: @PR24766_two_constants(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[CONDITION:%.*]] to i32
-; CHECK-NEXT:    switch i32 [[CONV]], label [[EPILOG:%.*]] [
-; CHECK-NEXT:      i32 0, label [[SW1:%.*]]
-; CHECK-NEXT:      i32 1, label [[SW2:%.*]]
+; CHECK-NEXT:    switch i8 [[CONDITION:%.*]], label [[EPILOG:%.*]] [
+; CHECK-NEXT:      i8 0, label [[SW1:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw1:
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[X1:%.*]], [[X2:%.*]]
@@ -1128,11 +1125,10 @@ epilog:
 define i1 @PR24766_two_constants_two_var(i8 %x1, i8 %x2, i8 %condition) {
 ; CHECK-LABEL: @PR24766_two_constants_two_var(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[CONDITION:%.*]] to i32
-; CHECK-NEXT:    switch i32 [[CONV]], label [[EPILOG:%.*]] [
-; CHECK-NEXT:      i32 0, label [[SW1:%.*]]
-; CHECK-NEXT:      i32 1, label [[SW2:%.*]]
-; CHECK-NEXT:      i32 2, label [[SW3:%.*]]
+; CHECK-NEXT:    switch i8 [[CONDITION:%.*]], label [[EPILOG:%.*]] [
+; CHECK-NEXT:      i8 0, label [[SW1:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW2:%.*]]
+; CHECK-NEXT:      i8 2, label [[SW3:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw1:
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[X1:%.*]], [[X2:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/switch-zext-sext.ll b/llvm/test/Transforms/InstCombine/switch-zext-sext.ll
new file mode 100644
index 000000000000..c09441352acc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/switch-zext-sext.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt %s -passes=instcombine -S | FileCheck %s
+
+define i1 @test_switch_with_zext(i16 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: define i1 @test_switch_with_zext(
+; CHECK-SAME: i16 [[A:%.*]], i1 [[B:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i16 [[A]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:      i16 37, label [[SW_BB:%.*]]
+; CHECK-NEXT:      i16 38, label [[SW_BB]]
+; CHECK-NEXT:      i16 39, label [[SW_BB]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    ret i1 [[B]]
+; CHECK:       sw.default:
+; CHECK-NEXT:    ret i1 [[C]]
+;
+entry:
+  %a.ext = zext i16 %a to i32
+  switch i32 %a.ext, label %sw.default [
+  i32 37, label %sw.bb
+  i32 38, label %sw.bb
+  i32 39, label %sw.bb
+  ]
+
+sw.bb:
+  ret i1 %b
+sw.default:
+  ret i1 %c
+}
+
+define i1 @test_switch_with_sext(i16 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: define i1 @test_switch_with_sext(
+; CHECK-SAME: i16 [[A:%.*]], i1 [[B:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i16 [[A]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:      i16 37, label [[SW_BB:%.*]]
+; CHECK-NEXT:      i16 38, label [[SW_BB]]
+; CHECK-NEXT:      i16 39, label [[SW_BB]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    ret i1 [[B]]
+; CHECK:       sw.default:
+; CHECK-NEXT:    ret i1 [[C]]
+;
+entry:
+  %a.ext = sext i16 %a to i32
+  switch i32 %a.ext, label %sw.default [
+  i32 37, label %sw.bb
+  i32 38, label %sw.bb
+  i32 39, label %sw.bb
+  ]
+
+sw.bb:
+  ret i1 %b
+sw.default:
+  ret i1 %c
+}
+
+; Negative tests
+
+define i1 @test_switch_with_zext_unreachable_case(i16 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: define i1 @test_switch_with_zext_unreachable_case(
+; CHECK-SAME: i16 [[A:%.*]], i1 [[B:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext i16 [[A]] to i32
+; CHECK-NEXT:    switch i32 [[A_EXT]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:      i32 37, label [[SW_BB:%.*]]
+; CHECK-NEXT:      i32 38, label [[SW_BB]]
+; CHECK-NEXT:      i32 39, label [[SW_BB]]
+; CHECK-NEXT:      i32 65537, label [[SW_BB]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    ret i1 [[B]]
+; CHECK:       sw.default:
+; CHECK-NEXT:    ret i1 [[C]]
+;
+entry:
+  %a.ext = zext i16 %a to i32
+  switch i32 %a.ext, label %sw.default [
+  i32 37, label %sw.bb
+  i32 38, label %sw.bb
+  i32 39, label %sw.bb
+  i32 65537, label %sw.bb
+  ]
+
+sw.bb:
+  ret i1 %b
+sw.default:
+  ret i1 %c
+}
+
+define i1 @test_switch_with_sext_unreachable_case(i16 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: define i1 @test_switch_with_sext_unreachable_case(
+; CHECK-SAME: i16 [[A:%.*]], i1 [[B:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i16 [[A]] to i32
+; CHECK-NEXT:    switch i32 [[A_EXT]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:      i32 37, label [[SW_BB:%.*]]
+; CHECK-NEXT:      i32 38, label [[SW_BB]]
+; CHECK-NEXT:      i32 39, label [[SW_BB]]
+; CHECK-NEXT:      i32 -65537, label [[SW_BB]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    ret i1 [[B]]
+; CHECK:       sw.default:
+; CHECK-NEXT:    ret i1 [[C]]
+;
+entry:
+  %a.ext = sext i16 %a to i32
+  switch i32 %a.ext, label %sw.default [
+  i32 37, label %sw.bb
+  i32 38, label %sw.bb
+  i32 39, label %sw.bb
+  i32 -65537, label %sw.bb
+  ]
+
+sw.bb:
+  ret i1 %b
+sw.default:
+  ret i1 %c
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/switch-sext.ll b/llvm/test/Transforms/PhaseOrdering/switch-sext.ll
new file mode 100644
index 000000000000..e39771b629e0
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/switch-sext.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -passes='default<O3>' < %s | FileCheck %s
+
+define i8 @test_switch_with_sext_phi(i8 %code) {
+; CHECK-LABEL: define noundef i8 @test_switch_with_sext_phi(
+; CHECK-SAME: i8 [[CODE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[CODE]], label [[SW_EPILOG:%.*]] [
+; CHECK-NEXT:      i8 76, label [[SW_BB3:%.*]]
+; CHECK-NEXT:      i8 108, label [[SW_BB2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb2:
+; CHECK-NEXT:    br label [[SW_EPILOG]]
+; CHECK:       sw.bb3:
+; CHECK-NEXT:    br label [[SW_EPILOG]]
+; CHECK:       sw.epilog:
+; CHECK-NEXT:    [[PEP_CODE:%.*]] = phi i8 [ 81, [[SW_BB3]] ], [ 113, [[SW_BB2]] ], [ [[CODE]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i8 [[PEP_CODE]]
+;
+entry:
+  %conv = sext i8 %code to i32
+  switch i32 %conv, label %sw.default [
+  i32 105, label %sw.epilog
+  i32 73, label %sw.bb1
+  i32 108, label %sw.bb2
+  i32 76, label %sw.bb3
+  i32 63, label %sw.bb4
+  ]
+
+sw.bb1:
+  br label %sw.epilog
+
+sw.bb2:
+  br label %sw.epilog
+
+sw.bb3:
+  br label %sw.epilog
+
+sw.bb4:
+  br label %sw.epilog
+
+sw.default:
+  br label %sw.epilog
+
+sw.epilog:
+  %pep_code = phi i8 [ %code, %sw.default ], [ 63, %sw.bb4 ], [ 81, %sw.bb3 ], [ 113, %sw.bb2 ], [ 73, %sw.bb1 ], [ 105, %entry ]
+  ret i8 %pep_code
+}
diff --git a/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml b/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml
new file mode 100644
index 000000000000..1327cc26ee9f
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml
@@ -0,0 +1,355 @@
+# RUN: yaml2obj %s -o %t.o
+# RUN: llvm-dwarfdump -arch arm64 --debug-line --verify %t.o 2>&1 | FileCheck %s
+
+# CHECK-NOT: error: .debug_line[0x{{[0-9a-f]+}}][0] has invalid file index 1 (valid values are [1,0]):
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x100000C
+  cpusubtype:      0x0
+  filetype:        0x1
+  ncmds:           4
+  sizeofcmds:      1000
+  flags:           0x2000
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         872
+    segname:         ''
+    vmaddr:          0
+    vmsize:          635
+    fileoff:         1032
+    filesize:        635
+    maxprot:         7
+    initprot:        7
+    nsects:          10
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0
+        size:            4
+        offset:          0x408
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         C0035FD6
+      - sectname:        __debug_abbrev
+        segname:         __DWARF
+        addr:            0x4
+        size:            50
+        offset:          0x40C
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+      - sectname:        __debug_info
+        segname:         __DWARF
+        addr:            0x36
+        size:            76
+        offset:          0x43E
+        align:           0
+        reloff:          0x688
+        nreloc:          2
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        relocations:
+          - address:         0x33
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x26
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+      - sectname:        __debug_str
+        segname:         __DWARF
+        addr:            0x82
+        size:            196
+        offset:          0x48A
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+      - sectname:        __apple_names
+        segname:         __DWARF
+        addr:            0x146
+        size:            88
+        offset:          0x54E
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         485341480100000002000000020000000C000000000000000100000001000600FFFFFFFF000000008973880BEB28616A3800000048000000B8000000010000003200000000000000BC000000010000003200000000000000
+      - sectname:        __apple_objc
+        segname:         __DWARF
+        addr:            0x19E
+        size:            36
+        offset:          0x5A6
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         485341480100000001000000000000000C000000000000000100000001000600FFFFFFFF
+      - sectname:        __apple_namespac
+        segname:         __DWARF
+        addr:            0x1C2
+        size:            36
+        offset:          0x5CA
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         485341480100000001000000000000000C000000000000000100000001000600FFFFFFFF
+      - sectname:        __apple_types
+        segname:         __DWARF
+        addr:            0x1E6
+        size:            44
+        offset:          0x5EE
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         48534148010000000100000000000000140000000000000003000000010006000300050004000B00FFFFFFFF
+      - sectname:        __compact_unwind
+        segname:         __LD
+        addr:            0x218
+        size:            32
+        offset:          0x620
+        align:           3
+        reloff:          0x698
+        nreloc:          1
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '0000000000000000040000000000000200000000000000000000000000000000'
+        relocations:
+          - address:         0x0
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+      - sectname:        __debug_line
+        segname:         __DWARF
+        addr:            0x238
+        size:            67
+        offset:          0x640
+        align:           0
+        reloff:          0x6A0
+        nreloc:          1
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        relocations:
+          - address:         0x35
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         24
+    platform:        1
+    minos:           917504
+    sdk:             918528
+    ntools:          0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          1704
+    nsyms:           3
+    stroff:          1752
+    strsize:         24
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       2
+    iextdefsym:      2
+    nextdefsym:      1
+    iundefsym:       3
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:
+  NameList:
+    - n_strx:          16
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          10
+      n_type:          0xE
+      n_sect:          9
+      n_desc:          0
+      n_value:         536
+    - n_strx:          1
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+  StringTable:
+    - ''
+    - __Z3foov
+    - ltmp1
+    - ltmp0
+    - ''
+    - ''
+DWARF:
+  debug_str:
+    - 'Apple clang version 16.0.0 (clang-1600.0.9.14)'
+    - '/tmp/test.cpp'
+    - '/Library/Developer/CommandLineTools/SDKs/MacOSX14.4.sdk'
+    - MacOSX14.4.sdk
+    - '/Users/shubham/Development/llvm-project/build_ninja'
+    - foo
+    - _Z3foov
+  debug_abbrev:
+    - ID:              0
+      Table:
+        - Code:            0x1
+          Tag:             DW_TAG_compile_unit
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_producer
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_language
+              Form:            DW_FORM_data2
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_LLVM_sysroot
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_APPLE_sdk
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_stmt_list
+              Form:            DW_FORM_sec_offset
+            - Attribute:       DW_AT_comp_dir
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_data4
+        - Code:            0x2
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_data4
+            - Attribute:       DW_AT_APPLE_omit_frame_ptr
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_frame_base
+              Form:            DW_FORM_exprloc
+            - Attribute:       DW_AT_linkage_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_external
+              Form:            DW_FORM_flag_present
+  debug_info:
+    - Length:          0x48
+      Version:         4
+      AbbrevTableID:   0
+      AbbrOffset:      0x0
+      AddrSize:        8
+      Entries:
+        - AbbrCode:        0x1
+          Values:
+            - Value:           0x0
+            - Value:           0x4
+            - Value:           0x2F
+            - Value:           0x3D
+            - Value:           0x75
+            - Value:           0x0
+            - Value:           0x84
+            - Value:           0x0
+            - Value:           0x4
+        - AbbrCode:        0x2
+          Values:
+            - Value:           0x0
+            - Value:           0x4
+            - Value:           0x1
+            - Value:           0x1
+              BlockData:       [ 0x6F ]
+            - Value:           0xBC
+            - Value:           0xB8
+            - Value:           0x1
+            - Value:           0x1
+            - Value:           0x1
+        - AbbrCode:        0x0
+  debug_line:
+    - Length:          38
+      Version:         4
+      PrologueLength:  29
+      MinInstLength:   1
+      MaxOpsPerInst:   1
+      DefaultIsStmt:   1
+      LineBase:        251
+      LineRange:       14
+      OpcodeBase:      13
+      StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
+      IncludeDirs:
+        - '/tmp'
+      Files:
+        - Name:            ''
+          DirIdx:          10
+          ModTime:         11
+          Length:          12
+      Opcodes:
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+...
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
index 5f08c67bfc89..dee7af5fd520 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
@@ -356,7 +356,11 @@ private:
       if (ChildExitCode == 0) {
         // The child exited succesfully, read counter values and return
         // success
-        CounterValues[0] = Counter->read();
+        auto CounterValueOrErr = Counter->readOrError();
+        if (!CounterValueOrErr)
+          return CounterValueOrErr.takeError();
+        CounterValues = std::move(*CounterValueOrErr);
+
         return Error::success();
       }
       // The child exited, but not successfully
diff --git a/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp b/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
index 3ff1745e9e06..314de1ec3236 100644
--- a/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
+++ b/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
@@ -148,17 +148,6 @@ void Counter::stop() {
     ioctl(FileDescriptor, PERF_EVENT_IOC_DISABLE, 0);
 }
 
-int64_t Counter::read() const {
-  auto ValueOrError = readOrError();
-  if (ValueOrError) {
-    if (!ValueOrError.get().empty())
-      return ValueOrError.get()[0];
-    errs() << "Counter has no reading\n";
-  } else
-    errs() << ValueOrError.takeError() << "\n";
-  return -1;
-}
-
 llvm::Expected<llvm::SmallVector<int64_t, 4>>
 Counter::readOrError(StringRef /*unused*/) const {
   int64_t Count = 0;
@@ -187,8 +176,6 @@ void Counter::start() {}
 
 void Counter::stop() {}
 
-int64_t Counter::read() const { return 42; }
-
 llvm::Expected<llvm::SmallVector<int64_t, 4>>
 Counter::readOrError(StringRef /*unused*/) const {
   if (IsDummyEvent) {
diff --git a/llvm/tools/llvm-exegesis/lib/PerfHelper.h b/llvm/tools/llvm-exegesis/lib/PerfHelper.h
index a50974f0a67b..894aac1f197e 100644
--- a/llvm/tools/llvm-exegesis/lib/PerfHelper.h
+++ b/llvm/tools/llvm-exegesis/lib/PerfHelper.h
@@ -95,9 +95,6 @@ public:
   /// Stops the measurement of the event.
   void stop();
 
-  /// Returns the current value of the counter or -1 if it cannot be read.
-  int64_t read() const;
-
   /// Returns the current value of the counter or error if it cannot be read.
   /// FunctionBytes: The benchmark function being executed.
   /// This is used to filter out the measurements to ensure they are only
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp
index 6c558b59be98..616f7bac54bc 100644
--- a/llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp
+++ b/llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp
@@ -46,6 +46,15 @@ MATCHER(EqMCInst, "") {
   return true;
 }
 
+MATCHER(EqRegValue, "") {
+  const RegisterValue Lhs = get<0>(arg);
+  const RegisterValue Rhs = get<1>(arg);
+  if (Lhs.Register != Rhs.Register || Lhs.Value != Rhs.Value)
+    return false;
+
+  return true;
+}
+
 namespace {
 
 TEST(BenchmarkResultTest, WriteToAndReadFromDisk) {
@@ -120,6 +129,8 @@ TEST(BenchmarkResultTest, WriteToAndReadFromDisk) {
     EXPECT_THAT(FromDisk.Key.Instructions,
                 Pointwise(EqMCInst(), ToDisk.Key.Instructions));
     EXPECT_EQ(FromDisk.Key.Config, ToDisk.Key.Config);
+    EXPECT_THAT(FromDisk.Key.RegisterInitialValues,
+                Pointwise(EqRegValue(), ToDisk.Key.RegisterInitialValues));
     EXPECT_EQ(FromDisk.Mode, ToDisk.Mode);
     EXPECT_EQ(FromDisk.CpuName, ToDisk.CpuName);
     EXPECT_EQ(FromDisk.LLVMTriple, ToDisk.LLVMTriple);
@@ -137,6 +148,8 @@ TEST(BenchmarkResultTest, WriteToAndReadFromDisk) {
     EXPECT_THAT(FromDisk.Key.Instructions,
                 Pointwise(EqMCInst(), ToDisk.Key.Instructions));
     EXPECT_EQ(FromDisk.Key.Config, ToDisk.Key.Config);
+    EXPECT_THAT(FromDisk.Key.RegisterInitialValues,
+                Pointwise(EqRegValue(), ToDisk.Key.RegisterInitialValues));
     EXPECT_EQ(FromDisk.Mode, ToDisk.Mode);
     EXPECT_EQ(FromDisk.CpuName, ToDisk.CpuName);
     EXPECT_EQ(FromDisk.LLVMTriple, ToDisk.LLVMTriple);
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
index 36d5b103e4cf..ac0dd8674355 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
@@ -63,6 +63,7 @@ executable("llvm-tblgen") {
     "InfoByHwMode.cpp",
     "InstrDocsEmitter.cpp",
     "InstrInfoEmitter.cpp",
+    "MacroFusionPredicatorEmitter.cpp",
     "OptEmitter.cpp",
     "OptParserEmitter.cpp",
     "OptRSTEmitter.cpp",
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 7d2bb2980255..f222831eefd7 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -540,6 +540,7 @@ libc_support_library(
         ":__support_cpp_bit",
         ":__support_cpp_limits",
         ":__support_cpp_optional",
+        ":__support_cpp_string_view",
         ":__support_ctype_utils",
         ":__support_fputil_dyadic_float",
         ":__support_fputil_fenv_impl",
@@ -1874,11 +1875,29 @@ libc_math_function(name = "llroundf")
 
 libc_math_function(name = "llroundl")
 
-libc_math_function(name = "nan")
+libc_math_function(
+    name = "nan",
+    additional_deps = [
+        ":__support_str_to_float",
+        ":errno",
+    ],
+)
 
-libc_math_function(name = "nanf")
+libc_math_function(
+    name = "nanf",
+    additional_deps = [
+        ":__support_str_to_float",
+        ":errno",
+    ],
+)
 
-libc_math_function(name = "nanl")
+libc_math_function(
+    name = "nanl",
+    additional_deps = [
+        ":__support_str_to_float",
+        ":errno",
+    ],
+)
 
 libc_math_function(name = "nearbyint")